2024-11-13 16:02:31,782 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-13 16:02:31,798 main DEBUG Took 0.012945 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging
2024-11-13 16:02:31,798 main DEBUG PluginManager 'Core' found 129 plugins
2024-11-13 16:02:31,799 main DEBUG PluginManager 'Level' found 0 plugins
2024-11-13 16:02:31,800 main DEBUG PluginManager 'Lookup' found 16 plugins
2024-11-13 16:02:31,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,809 main DEBUG PluginManager 'TypeConverter' found 26 plugins
2024-11-13 16:02:31,821 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,823 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,824 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,826 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,827 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,829 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,830 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,832 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,833 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,834 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig].
2024-11-13 16:02:31,836 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,836 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger].
2024-11-13 16:02:31,838 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null)
2024-11-13 16:02:31,840 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin].
2024-11-13 16:02:31,842 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root})
2024-11-13 16:02:31,843 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout].
2024-11-13 16:02:31,844 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null")
2024-11-13 16:02:31,845 main DEBUG PluginManager 'Converter' found 47 plugins
2024-11-13 16:02:31,855 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender].
2024-11-13 16:02:31,857 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={})
2024-11-13 16:02:31,859 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR
2024-11-13 16:02:31,859 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin].
2024-11-13 16:02:31,859 main DEBUG createAppenders(={Console})
2024-11-13 16:02:31,860 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized
2024-11-13 16:02:31,860 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca
2024-11-13 16:02:31,861 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK.
2024-11-13 16:02:31,861 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1
2024-11-13 16:02:31,861 main DEBUG OutputStream closed
2024-11-13 16:02:31,862 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true
2024-11-13 16:02:31,862 main DEBUG Appender DefaultConsole-1 stopped with status true
2024-11-13 16:02:31,862 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK
2024-11-13 16:02:31,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6
2024-11-13 16:02:31,927 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger
2024-11-13 16:02:31,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector
2024-11-13 16:02:31,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=
2024-11-13 16:02:31,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory
2024-11-13 16:02:31,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
2024-11-13 16:02:31,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper
2024-11-13 16:02:31,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j
2024-11-13 16:02:31,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl
2024-11-13 16:02:31,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans
2024-11-13 16:02:31,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase
2024-11-13 16:02:31,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop
2024-11-13 16:02:31,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers
2024-11-13 16:02:31,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices
2024-11-13 16:02:31,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig
2024-11-13 16:02:31,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel
2024-11-13 16:02:31,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore
2024-11-13 16:02:31,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console
2024-11-13 16:02:31,935 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-13 16:02:31,936 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null
2024-11-13 16:02:31,936 main DEBUG Shutdown hook enabled. Registering a new one.
2024-11-13 16:02:31,936 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK.
2024-11-13T16:02:32,199 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9
2024-11-13 16:02:32,201 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED)
2024-11-13 16:02:32,202 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps.
2024-11-13T16:02:32,210 INFO  [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins
2024-11-13T16:02:32,243 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=11854
2024-11-13T16:02:32,246 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:02:32,263 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099, deleteOnExit=true
2024-11-13T16:02:32,263 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:02:32,264 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/test.cache.data in system properties and HBase conf
2024-11-13T16:02:32,265 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:02:32,265 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:02:32,266 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:02:32,266 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:02:32,267 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:02:32,365 WARN  [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
2024-11-13T16:02:32,473 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:02:32,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:02:32,476 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:02:32,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:02:32,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:02:32,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:02:32,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:02:32,478 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:02:32,479 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:02:32,479 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:02:32,479 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:02:32,480 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:02:32,480 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:02:32,480 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:02:32,481 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:02:32,950 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:02:33,284 WARN  [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
2024-11-13T16:02:33,361 INFO  [Time-limited test {}] log.Log(170): Logging initialized @2245ms to org.eclipse.jetty.util.log.Slf4jLog
2024-11-13T16:02:33,433 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:02:33,499 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:02:33,519 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:02:33,519 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:02:33,520 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:02:33,533 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:02:33,535 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:02:33,536 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:02:33,732 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/java.io.tmpdir/jetty-localhost-41955-hadoop-hdfs-3_4_1-tests_jar-_-any-3279793394074452705/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:02:33,739 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:41955}
2024-11-13T16:02:33,739 INFO  [Time-limited test {}] server.Server(415): Started @2625ms
2024-11-13T16:02:33,763 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:02:34,130 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:02:34,138 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:02:34,139 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:02:34,139 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:02:34,139 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:02:34,140 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:02:34,141 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:02:34,268 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/java.io.tmpdir/jetty-localhost-41859-hadoop-hdfs-3_4_1-tests_jar-_-any-15125705619611085036/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:02:34,269 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:41859}
2024-11-13T16:02:34,270 INFO  [Time-limited test {}] server.Server(415): Started @3155ms
2024-11-13T16:02:34,328 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:02:34,455 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:02:34,461 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:02:34,463 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:02:34,463 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:02:34,463 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:02:34,467 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:02:34,468 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:02:34,592 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/java.io.tmpdir/jetty-localhost-44119-hadoop-hdfs-3_4_1-tests_jar-_-any-15454059529589583795/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:02:34,593 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:44119}
2024-11-13T16:02:34,593 INFO  [Time-limited test {}] server.Server(415): Started @3479ms
2024-11-13T16:02:34,596 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:02:34,829 WARN  [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data3/current/BP-662927508-172.17.0.3-1731513753042/current, will proceed with Du for space computation calculation, 
2024-11-13T16:02:34,829 WARN  [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data1/current/BP-662927508-172.17.0.3-1731513753042/current, will proceed with Du for space computation calculation, 
2024-11-13T16:02:34,829 WARN  [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data2/current/BP-662927508-172.17.0.3-1731513753042/current, will proceed with Du for space computation calculation, 
2024-11-13T16:02:34,833 WARN  [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data4/current/BP-662927508-172.17.0.3-1731513753042/current, will proceed with Du for space computation calculation, 
2024-11-13T16:02:34,929 WARN  [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:02:34,934 WARN  [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:02:35,011 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x696c619b91f64df5 with lease ID 0xaa41b2a919f05fc3: Processing first storage report for DS-d403b651-da1b-4e7c-91c2-8dee900cf82e from datanode DatanodeRegistration(127.0.0.1:43103, datanodeUuid=a6428107-5e4b-458a-b0bc-1fbf259c4138, infoPort=42829, infoSecurePort=0, ipcPort=34699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042)
2024-11-13T16:02:35,013 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x696c619b91f64df5 with lease ID 0xaa41b2a919f05fc3: from storage DS-d403b651-da1b-4e7c-91c2-8dee900cf82e node DatanodeRegistration(127.0.0.1:43103, datanodeUuid=a6428107-5e4b-458a-b0bc-1fbf259c4138, infoPort=42829, infoSecurePort=0, ipcPort=34699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0
2024-11-13T16:02:35,013 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x101b18016366a943 with lease ID 0xaa41b2a919f05fc4: Processing first storage report for DS-8da91f03-e6d6-4da2-b447-6e681c552dcb from datanode DatanodeRegistration(127.0.0.1:38019, datanodeUuid=3ce1265b-b697-4ebb-a6c9-1ada4c677405, infoPort=40491, infoSecurePort=0, ipcPort=39699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042)
2024-11-13T16:02:35,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x101b18016366a943 with lease ID 0xaa41b2a919f05fc4: from storage DS-8da91f03-e6d6-4da2-b447-6e681c552dcb node DatanodeRegistration(127.0.0.1:38019, datanodeUuid=3ce1265b-b697-4ebb-a6c9-1ada4c677405, infoPort=40491, infoSecurePort=0, ipcPort=39699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:02:35,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x696c619b91f64df5 with lease ID 0xaa41b2a919f05fc3: Processing first storage report for DS-1a9b74d1-c1a7-45cd-9bbe-c8ee3370181d from datanode DatanodeRegistration(127.0.0.1:43103, datanodeUuid=a6428107-5e4b-458a-b0bc-1fbf259c4138, infoPort=42829, infoSecurePort=0, ipcPort=34699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042)
2024-11-13T16:02:35,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x696c619b91f64df5 with lease ID 0xaa41b2a919f05fc3: from storage DS-1a9b74d1-c1a7-45cd-9bbe-c8ee3370181d node DatanodeRegistration(127.0.0.1:43103, datanodeUuid=a6428107-5e4b-458a-b0bc-1fbf259c4138, infoPort=42829, infoSecurePort=0, ipcPort=34699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:02:35,014 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x101b18016366a943 with lease ID 0xaa41b2a919f05fc4: Processing first storage report for DS-5851a70a-5e35-4488-9efc-c717ef13fd6a from datanode DatanodeRegistration(127.0.0.1:38019, datanodeUuid=3ce1265b-b697-4ebb-a6c9-1ada4c677405, infoPort=40491, infoSecurePort=0, ipcPort=39699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042)
2024-11-13T16:02:35,015 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x101b18016366a943 with lease ID 0xaa41b2a919f05fc4: from storage DS-5851a70a-5e35-4488-9efc-c717ef13fd6a node DatanodeRegistration(127.0.0.1:38019, datanodeUuid=3ce1265b-b697-4ebb-a6c9-1ada4c677405, infoPort=40491, infoSecurePort=0, ipcPort=39699, storageInfo=lv=-57;cid=testClusterID;nsid=1291154431;c=1731513753042), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:02:35,031 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9
2024-11-13T16:02:35,121 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/zookeeper_0, clientPort=52585, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:02:35,132 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52585
2024-11-13T16:02:35,144 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:35,147 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:35,388 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:02:35,389 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:02:35,795 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424 with version=8
2024-11-13T16:02:35,795 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:02:35,894 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16
2024-11-13T16:02:36,140 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:02:36,151 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,151 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,158 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:02:36,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,158 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:02:36,302 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:02:36,375 INFO  [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl
2024-11-13T16:02:36,384 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout
2024-11-13T16:02:36,388 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:02:36,413 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17426 (auto-detected)
2024-11-13T16:02:36,414 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected)
2024-11-13T16:02:36,432 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41613
2024-11-13T16:02:36,454 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41613 connecting to ZooKeeper ensemble=127.0.0.1:52585
2024-11-13T16:02:36,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416130x0, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:02:36,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41613-0x100a606b5f90000 connected
2024-11-13T16:02:36,524 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:36,527 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:36,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:02:36,544 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424, hbase.cluster.distributed=false
2024-11-13T16:02:36,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:02:36,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41613
2024-11-13T16:02:36,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41613
2024-11-13T16:02:36,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41613
2024-11-13T16:02:36,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41613
2024-11-13T16:02:36,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41613
2024-11-13T16:02:36,701 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:02:36,703 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,704 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,704 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:02:36,704 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:02:36,705 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:02:36,708 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:02:36,711 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:02:36,712 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40133
2024-11-13T16:02:36,714 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40133 connecting to ZooKeeper ensemble=127.0.0.1:52585
2024-11-13T16:02:36,716 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:36,720 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:36,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401330x0, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:02:36,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40133-0x100a606b5f90001 connected
2024-11-13T16:02:36,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:02:36,732 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:02:36,741 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:02:36,743 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:02:36,748 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:02:36,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40133
2024-11-13T16:02:36,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40133
2024-11-13T16:02:36,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40133
2024-11-13T16:02:36,752 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40133
2024-11-13T16:02:36,752 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40133
2024-11-13T16:02:36,767 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:41613
2024-11-13T16:02:36,768 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,41613,1731513755952
2024-11-13T16:02:36,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:02:36,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:02:36,777 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,41613,1731513755952
2024-11-13T16:02:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:02:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:36,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:36,804 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:02:36,810 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,41613,1731513755952 from backup master directory
2024-11-13T16:02:36,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:02:36,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,41613,1731513755952
2024-11-13T16:02:36,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:02:36,820 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:02:36,820 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,41613,1731513755952
2024-11-13T16:02:36,823 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0
2024-11-13T16:02:36,824 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0
2024-11-13T16:02:36,887 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase.id] with ID: 53603426-55d6-41ff-b425-37ce0d19e0c4
2024-11-13T16:02:36,887 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/.tmp/hbase.id
2024-11-13T16:02:36,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:02:36,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:02:36,900 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/.tmp/hbase.id]:[hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase.id]
2024-11-13T16:02:36,947 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:36,953 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:02:36,972 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms.
2024-11-13T16:02:36,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:36,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:36,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:02:36,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:02:37,009 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:02:37,011 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:02:37,017 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:02:37,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:02:37,052 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:02:37,069 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store
2024-11-13T16:02:37,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:02:37,091 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:02:37,096 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure.
2024-11-13T16:02:37,099 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:37,101 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:02:37,101 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:02:37,101 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:02:37,102 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:02:37,103 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:02:37,103 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:02:37,104 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513757101Disabling compacts and flushes for region at 1731513757101Disabling writes for close at 1731513757103 (+2 ms)Writing region close event to WAL at 1731513757103Closed at 1731513757103
2024-11-13T16:02:37,106 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/.initializing
2024-11-13T16:02:37,106 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/WALs/c4c650e80f58,41613,1731513755952
2024-11-13T16:02:37,128 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C41613%2C1731513755952, suffix=, logDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/WALs/c4c650e80f58,41613,1731513755952, archiveDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/oldWALs, maxLogs=10
2024-11-13T16:02:37,140 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41613%2C1731513755952.1731513757135
2024-11-13T16:02:37,164 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/WALs/c4c650e80f58,41613,1731513755952/c4c650e80f58%2C41613%2C1731513755952.1731513757135
2024-11-13T16:02:37,174 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40491:40491),(127.0.0.1/127.0.0.1:42829:42829)]
2024-11-13T16:02:37,176 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:02:37,176 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:37,179 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,180 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,219 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:02:37,250 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:02:37,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,257 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:37,258 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:02:37,262 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:02:37,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,263 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:02:37,263 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:02:37,266 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:02:37,266 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,267 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:02:37,268 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:02:37,271 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:02:37,271 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,272 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:02:37,272 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,276 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,277 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,282 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,283 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,286 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:02:37,290 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:02:37,294 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:02:37,295 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=739744, jitterRate=-0.05936744809150696}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:02:37,301 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513757192Initializing all the Stores at 1731513757195 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513757195Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513757196 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513757196Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513757196Cleaning up temporary data from old regions at 1731513757283 (+87 ms)Region opened successfully at 1731513757301 (+18 ms)
2024-11-13T16:02:37,303 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:02:37,339 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9febb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:02:37,371 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:02:37,383 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:02:37,383 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:02:37,386 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:02:37,388 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec
2024-11-13T16:02:37,392 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec
2024-11-13T16:02:37,393 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:02:37,418 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:02:37,427 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:02:37,430 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:02:37,432 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:02:37,434 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:02:37,435 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:02:37,438 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:02:37,441 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:02:37,444 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:02:37,446 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:02:37,448 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:02:37,465 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:02:37,467 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:02:37,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:02:37,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:02:37,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,474 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,41613,1731513755952, sessionid=0x100a606b5f90000, setting cluster-up flag (Was=false)
2024-11-13T16:02:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,495 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:02:37,497 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,41613,1731513755952
2024-11-13T16:02:37,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:37,509 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:02:37,510 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,41613,1731513755952
2024-11-13T16:02:37,516 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:02:37,557 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(746): ClusterId : 53603426-55d6-41ff-b425-37ce0d19e0c4
2024-11-13T16:02:37,559 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:02:37,564 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:02:37,565 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:02:37,568 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:02:37,568 DEBUG [RS:0;c4c650e80f58:40133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c49bcf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:02:37,583 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:40133
2024-11-13T16:02:37,586 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:02:37,587 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:02:37,587 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:02:37,589 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,41613,1731513755952 with port=40133, startcode=1731513756662
2024-11-13T16:02:37,591 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:02:37,601 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:02:37,603 DEBUG [RS:0;c4c650e80f58:40133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:02:37,611 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:02:37,619 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,41613,1731513755952
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:02:37,629 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,630 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:02:37,630 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,631 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731513787631
2024-11-13T16:02:37,632 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:02:37,633 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:02:37,636 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:02:37,636 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:02:37,637 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:02:37,637 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:02:37,638 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:02:37,638 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:02:37,638 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,643 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,644 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:02:37,646 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:02:37,646 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:02:37,644 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:02:37,648 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:02:37,649 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:02:37,651 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513757650,5,FailOnTimeoutGroup]
2024-11-13T16:02:37,652 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513757651,5,FailOnTimeoutGroup]
2024-11-13T16:02:37,652 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,652 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:02:37,653 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,655 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,666 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:02:37,667 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:02:37,670 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:02:37,670 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424
2024-11-13T16:02:37,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:02:37,684 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:02:37,697 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55071, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:02:37,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:37,706 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41613 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,709 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41613 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,711 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:02:37,725 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:02:37,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,726 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:37,727 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:02:37,730 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:02:37,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,732 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:37,732 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:02:37,736 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:02:37,736 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,742 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424
2024-11-13T16:02:37,743 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38585
2024-11-13T16:02:37,743 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:02:37,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:37,746 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:02:37,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:02:37,749 DEBUG [RS:0;c4c650e80f58:40133 {}] zookeeper.ZKUtil(111): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,749 WARN  [RS:0;c4c650e80f58:40133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:02:37,750 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:02:37,750 INFO  [RS:0;c4c650e80f58:40133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:02:37,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:37,751 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,751 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:37,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:02:37,752 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,40133,1731513756662]
2024-11-13T16:02:37,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740
2024-11-13T16:02:37,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740
2024-11-13T16:02:37,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:02:37,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:02:37,760 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:02:37,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:02:37,768 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:02:37,769 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=775447, jitterRate=-0.013968273997306824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:02:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513757703Initializing all the Stores at 1731513757710 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513757710Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513757711 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513757711Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513757711Cleaning up temporary data from old regions at 1731513757759 (+48 ms)Region opened successfully at 1731513757773 (+14 ms)
2024-11-13T16:02:37,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:02:37,773 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:02:37,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:02:37,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:02:37,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:02:37,776 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:02:37,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513757773Disabling compacts and flushes for region at 1731513757773Disabling writes for close at 1731513757774 (+1 ms)Writing region close event to WAL at 1731513757776 (+2 ms)Closed at 1731513757776
2024-11-13T16:02:37,780 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:02:37,780 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:02:37,788 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:02:37,789 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:02:37,800 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:02:37,803 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:02:37,805 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:02:37,810 INFO  [RS:0;c4c650e80f58:40133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:02:37,810 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,811 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:02:37,817 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:02:37,818 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,818 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:02:37,819 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:02:37,820 DEBUG [RS:0;c4c650e80f58:40133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:02:37,821 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,821 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,822 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,822 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,822 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,822 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40133,1731513756662-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:02:37,843 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:02:37,846 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40133,1731513756662-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,846 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,846 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.Replication(171): c4c650e80f58,40133,1731513756662 started
2024-11-13T16:02:37,871 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:37,872 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,40133,1731513756662, RpcServer on c4c650e80f58/172.17.0.3:40133, sessionid=0x100a606b5f90001
2024-11-13T16:02:37,873 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:02:37,873 DEBUG [RS:0;c4c650e80f58:40133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,873 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,40133,1731513756662'
2024-11-13T16:02:37,874 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:02:37,875 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:02:37,876 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:02:37,876 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:02:37,876 DEBUG [RS:0;c4c650e80f58:40133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,40133,1731513756662
2024-11-13T16:02:37,876 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,40133,1731513756662'
2024-11-13T16:02:37,876 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:02:37,877 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:02:37,878 DEBUG [RS:0;c4c650e80f58:40133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:02:37,878 INFO  [RS:0;c4c650e80f58:40133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:02:37,878 INFO  [RS:0;c4c650e80f58:40133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:02:37,954 WARN  [c4c650e80f58:41613 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:02:37,986 INFO  [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C40133%2C1731513756662, suffix=, logDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662, archiveDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs, maxLogs=32
2024-11-13T16:02:37,989 INFO  [RS:0;c4c650e80f58:40133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513757988
2024-11-13T16:02:37,998 INFO  [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513757988
2024-11-13T16:02:38,000 DEBUG [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:02:38,207 DEBUG [c4c650e80f58:41613 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:02:38,219 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:38,225 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,40133,1731513756662, state=OPENING
2024-11-13T16:02:38,230 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:02:38,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:38,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:02:38,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:02:38,233 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:02:38,234 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:02:38,236 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,40133,1731513756662}]
2024-11-13T16:02:38,412 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:02:38,415 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38487, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:02:38,426 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:02:38,427 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:02:38,431 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C40133%2C1731513756662.meta, suffix=.meta, logDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662, archiveDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs, maxLogs=32
2024-11-13T16:02:38,434 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.meta.1731513758434.meta
2024-11-13T16:02:38,442 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.meta.1731513758434.meta
2024-11-13T16:02:38,447 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40491:40491),(127.0.0.1/127.0.0.1:42829:42829)]
2024-11-13T16:02:38,448 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:02:38,450 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:02:38,453 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:02:38,457 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:02:38,462 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:02:38,462 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:38,462 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:02:38,462 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:02:38,465 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:02:38,467 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:02:38,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:38,468 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:38,469 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:02:38,470 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:02:38,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:38,471 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:38,472 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:02:38,473 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:02:38,473 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:38,474 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:38,474 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:02:38,475 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:02:38,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:38,476 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:02:38,476 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:02:38,478 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740
2024-11-13T16:02:38,480 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740
2024-11-13T16:02:38,482 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:02:38,482 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:02:38,483 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:02:38,486 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:02:38,487 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=857531, jitterRate=0.09040714800357819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:02:38,488 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:02:38,489 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513758463Writing region info on filesystem at 1731513758463Initializing all the Stores at 1731513758465 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513758465Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513758465Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513758465Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513758465Cleaning up temporary data from old regions at 1731513758482 (+17 ms)Running coprocessor post-open hooks at 1731513758488 (+6 ms)Region opened successfully at 1731513758489 (+1 ms)
2024-11-13T16:02:38,495 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513758403
2024-11-13T16:02:38,507 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:02:38,507 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:02:38,509 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:38,511 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,40133,1731513756662, state=OPEN
2024-11-13T16:02:38,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:02:38,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:02:38,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:02:38,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:02:38,517 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:38,522 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:02:38,523 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,40133,1731513756662 in 281 msec
2024-11-13T16:02:38,529 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:02:38,529 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 736 msec
2024-11-13T16:02:38,531 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:02:38,531 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:02:38,552 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:02:38,553 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,40133,1731513756662, seqNum=-1]
2024-11-13T16:02:38,574 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:02:38,576 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58591, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:02:38,597 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0500 sec
2024-11-13T16:02:38,597 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513758597, completionTime=-1
2024-11-13T16:02:38,599 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:02:38,600 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:02:38,627 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:02:38,627 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731513818627
2024-11-13T16:02:38,627 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731513878627
2024-11-13T16:02:38,627 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec
2024-11-13T16:02:38,630 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,630 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,630 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,632 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:41613, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,632 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,632 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,639 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:02:38,659 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.838sec
2024-11-13T16:02:38,660 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:02:38,662 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:02:38,662 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:02:38,663 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:02:38,663 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:02:38,664 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:02:38,664 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:02:38,672 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:02:38,672 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:02:38,673 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41613,1731513755952-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:02:38,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61c64a9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:02:38,795 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false
2024-11-13T16:02:38,795 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512
2024-11-13T16:02:38,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,41613,-1 for getting cluster id
2024-11-13T16:02:38,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:02:38,815 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '53603426-55d6-41ff-b425-37ce0d19e0c4'
2024-11-13T16:02:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:02:38,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "53603426-55d6-41ff-b425-37ce0d19e0c4"

2024-11-13T16:02:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6914c27c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:02:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,41613,-1]
2024-11-13T16:02:38,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:02:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:02:38,826 INFO  [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:02:38,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231106e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:02:38,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:02:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,40133,1731513756662, seqNum=-1]
2024-11-13T16:02:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:02:38,840 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:02:38,860 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,41613,1731513755952
2024-11-13T16:02:38,861 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:02:38,868 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:02:38,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-13T16:02:38,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c4c650e80f58,41613,1731513755952
2024-11-13T16:02:38,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6e074fe8
2024-11-13T16:02:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-13T16:02:38,884 INFO  [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-13T16:02:38,886 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-13T16:02:38,886 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-13T16:02:38,889 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:02:38,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling
2024-11-13T16:02:38,900 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-13T16:02:38,902 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testSlowSyncLogRolling"
 procId is: 4
2024-11-13T16:02:38,903 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:38,906 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-13T16:02:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:02:38,954 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741835_1011 (size=389)
2024-11-13T16:02:38,955 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741835_1011 (size=389)
2024-11-13T16:02:38,958 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e815b968e737d852f3ba6866e69ae63b, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424
2024-11-13T16:02:38,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741836_1012 (size=72)
2024-11-13T16:02:38,970 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741836_1012 (size=72)
2024-11-13T16:02:39,371 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:39,372 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing e815b968e737d852f3ba6866e69ae63b, disabling compactions & flushes
2024-11-13T16:02:39,372 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,372 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,372 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b. after waiting 0 ms
2024-11-13T16:02:39,372 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,372 INFO  [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,372 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e815b968e737d852f3ba6866e69ae63b:
Waiting for close lock at 1731513759372Disabling compacts and flushes for region at 1731513759372Disabling writes for close at 1731513759372Writing region close event to WAL at 1731513759372Closed at 1731513759372
2024-11-13T16:02:39,374 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-13T16:02:39,379 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731513759374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731513759374"}]},"ts":"1731513759374"}
2024-11-13T16:02:39,385 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-13T16:02:39,387 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-13T16:02:39,389 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513759387"}]},"ts":"1731513759387"}
2024-11-13T16:02:39,394 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta
2024-11-13T16:02:39,396 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=e815b968e737d852f3ba6866e69ae63b, ASSIGN}]
2024-11-13T16:02:39,398 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=e815b968e737d852f3ba6866e69ae63b, ASSIGN
2024-11-13T16:02:39,400 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=e815b968e737d852f3ba6866e69ae63b, ASSIGN; state=OFFLINE, location=c4c650e80f58,40133,1731513756662; forceNewPlan=false, retain=false
2024-11-13T16:02:39,551 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e815b968e737d852f3ba6866e69ae63b, regionState=OPENING, regionLocation=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:39,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=e815b968e737d852f3ba6866e69ae63b, ASSIGN because future has completed
2024-11-13T16:02:39,558 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e815b968e737d852f3ba6866e69ae63b, server=c4c650e80f58,40133,1731513756662}]
2024-11-13T16:02:39,718 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,719 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e815b968e737d852f3ba6866e69ae63b, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:02:39,719 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,719 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:02:39,719 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,719 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,722 INFO  [StoreOpener-e815b968e737d852f3ba6866e69ae63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e815b968e737d852f3ba6866e69ae63b 
2024-11-13T16:02:39,724 INFO  [StoreOpener-e815b968e737d852f3ba6866e69ae63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e815b968e737d852f3ba6866e69ae63b columnFamilyName info
2024-11-13T16:02:39,724 DEBUG [StoreOpener-e815b968e737d852f3ba6866e69ae63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:02:39,726 INFO  [StoreOpener-e815b968e737d852f3ba6866e69ae63b-1 {}] regionserver.HStore(327): Store=e815b968e737d852f3ba6866e69ae63b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:02:39,726 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,727 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,728 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,728 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,728 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,731 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,734 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:02:39,735 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e815b968e737d852f3ba6866e69ae63b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=745817, jitterRate=-0.05164483189582825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:02:39,735 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:02:39,736 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e815b968e737d852f3ba6866e69ae63b:
Running coprocessor pre-open hook at 1731513759720Writing region info on filesystem at 1731513759720Initializing all the Stores at 1731513759721 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513759721Cleaning up temporary data from old regions at 1731513759728 (+7 ms)Running coprocessor post-open hooks at 1731513759735 (+7 ms)Region opened successfully at 1731513759736 (+1 ms)
2024-11-13T16:02:39,738 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b., pid=6, masterSystemTime=1731513759712
2024-11-13T16:02:39,742 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,742 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:39,743 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e815b968e737d852f3ba6866e69ae63b, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,40133,1731513756662
2024-11-13T16:02:39,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e815b968e737d852f3ba6866e69ae63b, server=c4c650e80f58,40133,1731513756662 because future has completed
2024-11-13T16:02:39,754 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-13T16:02:39,754 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e815b968e737d852f3ba6866e69ae63b, server=c4c650e80f58,40133,1731513756662 in 192 msec
2024-11-13T16:02:39,758 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-13T16:02:39,758 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=e815b968e737d852f3ba6866e69ae63b, ASSIGN in 358 msec
2024-11-13T16:02:39,760 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-13T16:02:39,760 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513759760"}]},"ts":"1731513759760"}
2024-11-13T16:02:39,763 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta
2024-11-13T16:02:39,765 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-13T16:02:39,769 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 873 msec
2024-11-13T16:02:43,967 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties
2024-11-13T16:02:44,020 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-13T16:02:44,021 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling'
2024-11-13T16:02:46,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:02:46,371 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-13T16:02:46,373 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-13T16:02:46,373 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-13T16:02:46,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:02:46,374 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-13T16:02:46,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-13T16:02:46,374 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-13T16:02:49,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41613 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:02:49,005 INFO  [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed
2024-11-13T16:02:49,008 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100
2024-11-13T16:02:49,014 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling
2024-11-13T16:02:49,014 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:02:49,015 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513769015
2024-11-13T16:02:49,023 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:02:49,023 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:02:49,023 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:02:49,024 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:02:49,024 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:02:49,024 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513757988 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513769015
2024-11-13T16:02:49,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:02:49,026 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513757988 is not closed yet, will try archiving it next time
2024-11-13T16:02:49,028 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741833_1009 (size=451)
2024-11-13T16:02:49,029 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741833_1009 (size=451)
2024-11-13T16:02:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b., hostname=c4c650e80f58,40133,1731513756662, seqNum=2]
2024-11-13T16:02:49,431 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513757988 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513757988
2024-11-13T16:03:01,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] regionserver.HRegion(8855): Flush requested on e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:03:01,075 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e815b968e737d852f3ba6866e69ae63b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:03:01,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/56c56b99ce7f48a4b95c410b86675b8e is 1080, key is row0001/info:/1731513769037/Put/seqid=0
2024-11-13T16:03:01,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741838_1014 (size=12509)
2024-11-13T16:03:01,144 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741838_1014 (size=12509)
2024-11-13T16:03:01,145 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/56c56b99ce7f48a4b95c410b86675b8e
2024-11-13T16:03:01,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/56c56b99ce7f48a4b95c410b86675b8e as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e
2024-11-13T16:03:01,204 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e, entries=7, sequenceid=11, filesize=12.2 K
2024-11-13T16:03:01,211 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 136ms, sequenceid=11, compaction requested=false
2024-11-13T16:03:01,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e815b968e737d852f3ba6866e69ae63b:

2024-11-13T16:03:05,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:03:09,085 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513789085
2024-11-13T16:03:09,294 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:09,294 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:09,294 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:09,294 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:09,294 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:09,295 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:09,295 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513769015 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513789085
2024-11-13T16:03:09,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:03:09,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513769015 is not closed yet, will try archiving it next time
2024-11-13T16:03:09,297 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741837_1013 (size=12399)
2024-11-13T16:03:09,298 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741837_1013 (size=12399)
2024-11-13T16:03:09,500 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:11,704 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:13,908 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:16,113 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:16,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] regionserver.HRegion(8855): Flush requested on e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:03:16,113 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e815b968e737d852f3ba6866e69ae63b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:03:16,315 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:16,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9f36c75eaf894c60acfdb7c8f098098c is 1080, key is row0008/info:/1731513783073/Put/seqid=0
2024-11-13T16:03:16,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741840_1016 (size=12509)
2024-11-13T16:03:16,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741840_1016 (size=12509)
2024-11-13T16:03:16,335 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9f36c75eaf894c60acfdb7c8f098098c
2024-11-13T16:03:16,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9f36c75eaf894c60acfdb7c8f098098c as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c
2024-11-13T16:03:16,356 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c, entries=7, sequenceid=21, filesize=12.2 K
2024-11-13T16:03:16,558 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:16,558 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 445ms, sequenceid=21, compaction requested=false
2024-11-13T16:03:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e815b968e737d852f3ba6866e69ae63b:

2024-11-13T16:03:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K
2024-11-13T16:03:16,558 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:03:16,559 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e because midkey is the same as first or last row
2024-11-13T16:03:18,318 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:18,708 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-13T16:03:18,708 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-13T16:03:20,522 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:20,524 WARN  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:20,525 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C40133%2C1731513756662:(num 1731513789085) roll requested
2024-11-13T16:03:20,525 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513800525
2024-11-13T16:03:20,733 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:20,734 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:20,734 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:20,734 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:20,734 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:20,734 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:20,735 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513789085 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513800525
2024-11-13T16:03:20,735 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:03:20,735 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513789085 is not closed yet, will try archiving it next time
2024-11-13T16:03:20,736 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513769015 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513769015
2024-11-13T16:03:20,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741839_1015 (size=7739)
2024-11-13T16:03:20,742 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741839_1015 (size=7739)
2024-11-13T16:03:22,726 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:24,719 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e815b968e737d852f3ba6866e69ae63b, had cached 0 bytes from a total of 25018
2024-11-13T16:03:24,930 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:27,134 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:29,338 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:31,340 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-13T16:03:31,341 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513811341
2024-11-13T16:03:35,027 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:03:36,349 INFO  [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:36,351 WARN  [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:36,351 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C40133%2C1731513756662:(num 1731513811341) roll requested
2024-11-13T16:03:36,352 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:36,352 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:36,352 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:36,352 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:36,352 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:36,352 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513800525 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513811341
2024-11-13T16:03:36,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:03:36,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513800525 is not closed yet, will try archiving it next time
2024-11-13T16:03:36,354 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513816354
2024-11-13T16:03:36,355 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741841_1017 (size=4753)
2024-11-13T16:03:36,357 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741841_1017 (size=4753)
2024-11-13T16:03:41,357 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:41,357 WARN  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] regionserver.HRegion(8855): Flush requested on e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:03:41,358 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e815b968e737d852f3ba6866e69ae63b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:03:41,363 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:41,363 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:43,358 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-13T16:03:46,360 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:46,360 WARN  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:46,360 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:46,360 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:46,361 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:46,361 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:46,361 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:46,361 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513811341 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513816354
2024-11-13T16:03:46,362 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42829:42829),(127.0.0.1/127.0.0.1:40491:40491)]
2024-11-13T16:03:46,362 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513811341 is not closed yet, will try archiving it next time
2024-11-13T16:03:46,362 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C40133%2C1731513756662:(num 1731513816354) roll requested
2024-11-13T16:03:46,363 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513826363
2024-11-13T16:03:46,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741842_1018 (size=1569)
2024-11-13T16:03:46,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741842_1018 (size=1569)
2024-11-13T16:03:46,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/a517986c8d484e1baedc508b0e895738 is 1080, key is row0015/info:/1731513798116/Put/seqid=0
2024-11-13T16:03:46,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741844_1020 (size=12509)
2024-11-13T16:03:46,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741844_1020 (size=12509)
2024-11-13T16:03:46,377 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/a517986c8d484e1baedc508b0e895738
2024-11-13T16:03:46,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/a517986c8d484e1baedc508b0e895738 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738
2024-11-13T16:03:46,396 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738, entries=7, sequenceid=31, filesize=12.2 K
2024-11-13T16:03:51,374 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5007 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:51,374 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5007 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:51,398 INFO  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:51,398 WARN  [FSHLog-0-hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424-prefix:c4c650e80f58,40133,1731513756662 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43103,DS-d403b651-da1b-4e7c-91c2-8dee900cf82e,DISK], DatanodeInfoWithStorage[127.0.0.1:38019,DS-8da91f03-e6d6-4da2-b447-6e681c552dcb,DISK]]
2024-11-13T16:03:51,398 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,398 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 10041ms, sequenceid=31, compaction requested=true
2024-11-13T16:03:51,398 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e815b968e737d852f3ba6866e69ae63b:

2024-11-13T16:03:51,399 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,399 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K
2024-11-13T16:03:51,399 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:03:51,399 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513816354 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513826363
2024-11-13T16:03:51,399 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e because midkey is the same as first or last row
2024-11-13T16:03:51,400 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40491:40491),(127.0.0.1/127.0.0.1:42829:42829)]
2024-11-13T16:03:51,400 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513816354 is not closed yet, will try archiving it next time
2024-11-13T16:03:51,400 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513789085 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513789085
2024-11-13T16:03:51,400 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C40133%2C1731513756662:(num 1731513831400) roll requested
2024-11-13T16:03:51,400 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513831400
2024-11-13T16:03:51,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741843_1019 (size=438)
2024-11-13T16:03:51,402 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741843_1019 (size=438)
2024-11-13T16:03:51,403 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513800525 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513800525
2024-11-13T16:03:51,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e815b968e737d852f3ba6866e69ae63b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:03:51,405 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513811341 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513811341
2024-11-13T16:03:51,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:03:51,407 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:03:51,409 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513816354 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513816354
2024-11-13T16:03:51,411 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:03:51,413 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HStore(1541): e815b968e737d852f3ba6866e69ae63b/info is initiating minor compaction (all files)
2024-11-13T16:03:51,414 INFO  [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e815b968e737d852f3ba6866e69ae63b/info in TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:03:51,414 INFO  [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738] into tmpdir=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp, totalSize=36.6 K
2024-11-13T16:03:51,416 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56c56b99ce7f48a4b95c410b86675b8e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731513769037
2024-11-13T16:03:51,417 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f36c75eaf894c60acfdb7c8f098098c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731513783073
2024-11-13T16:03:51,418 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] compactions.Compactor(225): Compacting a517986c8d484e1baedc508b0e895738, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731513798116
2024-11-13T16:03:51,423 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,425 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,425 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,425 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,426 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,426 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513826363 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513831400
2024-11-13T16:03:51,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741845_1021 (size=93)
2024-11-13T16:03:51,432 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741845_1021 (size=93)
2024-11-13T16:03:51,433 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513826363 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs/c4c650e80f58%2C40133%2C1731513756662.1731513826363
2024-11-13T16:03:51,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40491:40491),(127.0.0.1/127.0.0.1:42829:42829)]
2024-11-13T16:03:51,443 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40133%2C1731513756662.1731513831443
2024-11-13T16:03:51,469 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,470 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,470 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,470 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,470 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:03:51,470 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513831400 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/WALs/c4c650e80f58,40133,1731513756662/c4c650e80f58%2C40133%2C1731513756662.1731513831443
2024-11-13T16:03:51,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741846_1022 (size=1258)
2024-11-13T16:03:51,473 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741846_1022 (size=1258)
2024-11-13T16:03:51,474 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40491:40491),(127.0.0.1/127.0.0.1:42829:42829)]
2024-11-13T16:03:51,482 INFO  [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e815b968e737d852f3ba6866e69ae63b#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:03:51,483 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/0d263035fdab442986bc232cdc587b34 is 1080, key is row0001/info:/1731513769037/Put/seqid=0
2024-11-13T16:03:51,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741848_1024 (size=27710)
2024-11-13T16:03:51,497 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741848_1024 (size=27710)
2024-11-13T16:03:51,510 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/0d263035fdab442986bc232cdc587b34 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/0d263035fdab442986bc232cdc587b34
2024-11-13T16:03:51,535 INFO  [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e815b968e737d852f3ba6866e69ae63b/info of e815b968e737d852f3ba6866e69ae63b into 0d263035fdab442986bc232cdc587b34(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:03:51,536 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e815b968e737d852f3ba6866e69ae63b:

2024-11-13T16:03:51,538 INFO  [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b., storeName=e815b968e737d852f3ba6866e69ae63b/info, priority=13, startTime=1731513831401; duration=0sec
2024-11-13T16:03:51,538 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-13T16:03:51,538 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/0d263035fdab442986bc232cdc587b34 because midkey is the same as first or last row
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/0d263035fdab442986bc232cdc587b34 because midkey is the same as first or last row
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K
2024-11-13T16:03:51,539 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:03:51,540 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/0d263035fdab442986bc232cdc587b34 because midkey is the same as first or last row
2024-11-13T16:03:51,540 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:03:51,540 DEBUG [RS:0;c4c650e80f58:40133-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e815b968e737d852f3ba6866e69ae63b:info
2024-11-13T16:04:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40133 {}] regionserver.HRegion(8855): Flush requested on e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:04:03,470 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e815b968e737d852f3ba6866e69ae63b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:04:03,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9daab553fe354d5ebb387aa07e252616 is 1080, key is row0022/info:/1731513831445/Put/seqid=0
2024-11-13T16:04:03,487 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741849_1025 (size=12509)
2024-11-13T16:04:03,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741849_1025 (size=12509)
2024-11-13T16:04:03,489 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9daab553fe354d5ebb387aa07e252616
2024-11-13T16:04:03,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/9daab553fe354d5ebb387aa07e252616 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9daab553fe354d5ebb387aa07e252616
2024-11-13T16:04:03,508 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9daab553fe354d5ebb387aa07e252616, entries=7, sequenceid=42, filesize=12.2 K
2024-11-13T16:04:03,511 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 40ms, sequenceid=42, compaction requested=false
2024-11-13T16:04:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e815b968e737d852f3ba6866e69ae63b:

2024-11-13T16:04:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:03,511 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/0d263035fdab442986bc232cdc587b34 because midkey is the same as first or last row
2024-11-13T16:04:05,028 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:04:09,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e815b968e737d852f3ba6866e69ae63b, had cached 0 bytes from a total of 40219
2024-11-13T16:04:11,486 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:04:11,487 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:11,487 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:11,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:11,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:11,495 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:04:11,495 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:04:11,495 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=715057171, stopped=false
2024-11-13T16:04:11,496 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,41613,1731513755952
2024-11-13T16:04:11,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:11,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:11,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:11,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:11,498 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:11,499 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:11,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:11,499 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:11,499 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:11,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:11,499 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,40133,1731513756662' *****
2024-11-13T16:04:11,500 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:04:11,500 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:04:11,500 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:04:11,500 INFO  [RS:0;c4c650e80f58:40133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:04:11,500 INFO  [RS:0;c4c650e80f58:40133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:04:11,501 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(3091): Received CLOSE for e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:04:11,501 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,40133,1731513756662
2024-11-13T16:04:11,501 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:11,502 INFO  [RS:0;c4c650e80f58:40133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:40133.
2024-11-13T16:04:11,502 DEBUG [RS:0;c4c650e80f58:40133 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:11,502 DEBUG [RS:0;c4c650e80f58:40133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:11,502 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e815b968e737d852f3ba6866e69ae63b, disabling compactions & flushes
2024-11-13T16:04:11,502 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:04:11,502 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:04:11,502 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:04:11,502 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:04:11,502 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:04:11,502 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b. after waiting 0 ms
2024-11-13T16:04:11,502 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:04:11,502 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:04:11,502 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e815b968e737d852f3ba6866e69ae63b 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB
2024-11-13T16:04:11,503 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-13T16:04:11,503 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:04:11,503 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1325): Online Regions={e815b968e737d852f3ba6866e69ae63b=TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b., 1588230740=hbase:meta,,1.1588230740}
2024-11-13T16:04:11,503 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:04:11,503 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:04:11,503 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:04:11,503 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:04:11,503 DEBUG [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e815b968e737d852f3ba6866e69ae63b
2024-11-13T16:04:11,503 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB
2024-11-13T16:04:11,509 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/69a2481f92c5440c93a09a6721c3d759 is 1080, key is row0029/info:/1731513845476/Put/seqid=0
2024-11-13T16:04:11,516 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741850_1026 (size=8193)
2024-11-13T16:04:11,517 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741850_1026 (size=8193)
2024-11-13T16:04:11,517 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/69a2481f92c5440c93a09a6721c3d759
2024-11-13T16:04:11,528 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/.tmp/info/69a2481f92c5440c93a09a6721c3d759 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/69a2481f92c5440c93a09a6721c3d759
2024-11-13T16:04:11,532 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/info/ff28e928ccfe4858bb50cedaa84e2d1d is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b./info:regioninfo/1731513759743/Put/seqid=0
2024-11-13T16:04:11,536 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/69a2481f92c5440c93a09a6721c3d759, entries=3, sequenceid=48, filesize=8.0 K
2024-11-13T16:04:11,538 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 36ms, sequenceid=48, compaction requested=true
2024-11-13T16:04:11,542 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738] to archive
2024-11-13T16:04:11,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741851_1027 (size=7016)
2024-11-13T16:04:11,544 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741851_1027 (size=7016)
2024-11-13T16:04:11,545 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/info/ff28e928ccfe4858bb50cedaa84e2d1d
2024-11-13T16:04:11,545 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:04:11,549 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/archive/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/56c56b99ce7f48a4b95c410b86675b8e
2024-11-13T16:04:11,552 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/archive/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/9f36c75eaf894c60acfdb7c8f098098c
2024-11-13T16:04:11,554 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738 to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/archive/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/info/a517986c8d484e1baedc508b0e895738
2024-11-13T16:04:11,566 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c4c650e80f58:41613 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-13T16:04:11,572 WARN  [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [56c56b99ce7f48a4b95c410b86675b8e=12509, 9f36c75eaf894c60acfdb7c8f098098c=12509, a517986c8d484e1baedc508b0e895738=12509]
2024-11-13T16:04:11,572 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/ns/8f39d9426fa54438b5c64c94e5e258b1 is 43, key is default/ns:d/1731513758580/Put/seqid=0
2024-11-13T16:04:11,586 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/default/TestLogRolling-testSlowSyncLogRolling/e815b968e737d852f3ba6866e69ae63b/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1
2024-11-13T16:04:11,587 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741852_1028 (size=5153)
2024-11-13T16:04:11,588 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741852_1028 (size=5153)
2024-11-13T16:04:11,588 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/ns/8f39d9426fa54438b5c64c94e5e258b1
2024-11-13T16:04:11,589 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:04:11,590 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e815b968e737d852f3ba6866e69ae63b:
Waiting for close lock at 1731513851501Running coprocessor pre-close hooks at 1731513851502 (+1 ms)Disabling compacts and flushes for region at 1731513851502Disabling writes for close at 1731513851502Obtaining lock to block concurrent updates at 1731513851502Preparing flush snapshotting stores in e815b968e737d852f3ba6866e69ae63b at 1731513851502Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731513851503 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b. at 1731513851504 (+1 ms)Flushing e815b968e737d852f3ba6866e69ae63b/info: creating writer at 1731513851504Flushing e815b968e737d852f3ba6866e69ae63b/info: appending metadata at 1731513851508 (+4 ms)Flushing e815b968e737d852f3ba6866e69ae63b/info: closing flushed file at 1731513851509 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66e43588: reopening flushed file at 1731513851527 (+18 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for e815b968e737d852f3ba6866e69ae63b in 36ms, sequenceid=48, compaction requested=true at 1731513851538 (+11 ms)Writing region close event to WAL at 1731513851575 (+37 ms)Running coprocessor post-close hooks at 1731513851587 (+12 ms)Closed at 1731513851589 (+2 ms)
2024-11-13T16:04:11,590 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731513758885.e815b968e737d852f3ba6866e69ae63b.
2024-11-13T16:04:11,615 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/table/895b3056349346df8ad0aa39ae885e9c is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731513759760/Put/seqid=0
2024-11-13T16:04:11,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741853_1029 (size=5396)
2024-11-13T16:04:11,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741853_1029 (size=5396)
2024-11-13T16:04:11,622 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/table/895b3056349346df8ad0aa39ae885e9c
2024-11-13T16:04:11,630 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/info/ff28e928ccfe4858bb50cedaa84e2d1d as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/info/ff28e928ccfe4858bb50cedaa84e2d1d
2024-11-13T16:04:11,638 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/info/ff28e928ccfe4858bb50cedaa84e2d1d, entries=10, sequenceid=11, filesize=6.9 K
2024-11-13T16:04:11,639 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/ns/8f39d9426fa54438b5c64c94e5e258b1 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/ns/8f39d9426fa54438b5c64c94e5e258b1
2024-11-13T16:04:11,646 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/ns/8f39d9426fa54438b5c64c94e5e258b1, entries=2, sequenceid=11, filesize=5.0 K
2024-11-13T16:04:11,648 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/.tmp/table/895b3056349346df8ad0aa39ae885e9c as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/table/895b3056349346df8ad0aa39ae885e9c
2024-11-13T16:04:11,654 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/table/895b3056349346df8ad0aa39ae885e9c, entries=2, sequenceid=11, filesize=5.3 K
2024-11-13T16:04:11,656 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 153ms, sequenceid=11, compaction requested=false
2024-11-13T16:04:11,662 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-13T16:04:11,663 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:11,663 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:11,663 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513851503Running coprocessor pre-close hooks at 1731513851503Disabling compacts and flushes for region at 1731513851503Disabling writes for close at 1731513851503Obtaining lock to block concurrent updates at 1731513851503Preparing flush snapshotting stores in 1588230740 at 1731513851503Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731513851504 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731513851505 (+1 ms)Flushing 1588230740/info: creating writer at 1731513851505Flushing 1588230740/info: appending metadata at 1731513851531 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731513851531Flushing 1588230740/ns: creating writer at 1731513851554 (+23 ms)Flushing 1588230740/ns: appending metadata at 1731513851571 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731513851571Flushing 1588230740/table: creating writer at 1731513851598 (+27 ms)Flushing 1588230740/table: appending metadata at 1731513851614 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731513851614Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca569f3: reopening flushed file at 1731513851629 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc56f52: reopening flushed file at 1731513851638 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3681dcf6: reopening flushed file at 1731513851647 (+9 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 153ms, sequenceid=11, compaction requested=false at 1731513851656 (+9 ms)Writing region close event to WAL at 1731513851657 (+1 ms)Running coprocessor post-close hooks at 1731513851663 (+6 ms)Closed at 1731513851663
2024-11-13T16:04:11,663 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:11,703 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,40133,1731513756662; all regions closed.
2024-11-13T16:04:11,705 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,705 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,705 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,706 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,706 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,708 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741834_1010 (size=3066)
2024-11-13T16:04:11,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741834_1010 (size=3066)
2024-11-13T16:04:11,712 DEBUG [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs
2024-11-13T16:04:11,712 INFO  [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C40133%2C1731513756662.meta:.meta(num 1731513758434)
2024-11-13T16:04:11,712 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,712 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,713 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,713 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,713 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:11,715 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741847_1023 (size=12695)
2024-11-13T16:04:11,715 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741847_1023 (size=12695)
2024-11-13T16:04:11,719 DEBUG [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/oldWALs
2024-11-13T16:04:11,719 INFO  [RS:0;c4c650e80f58:40133 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C40133%2C1731513756662:(num 1731513831443)
2024-11-13T16:04:11,719 DEBUG [RS:0;c4c650e80f58:40133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:11,719 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:11,719 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:11,720 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:11,720 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:11,720 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:11,720 INFO  [RS:0;c4c650e80f58:40133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40133
2024-11-13T16:04:11,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:11,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,40133,1731513756662
2024-11-13T16:04:11,724 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:11,725 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,40133,1731513756662]
2024-11-13T16:04:11,728 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,40133,1731513756662 already deleted, retry=false
2024-11-13T16:04:11,728 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,40133,1731513756662 expired; onlineServers=0
2024-11-13T16:04:11,729 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,41613,1731513755952' *****
2024-11-13T16:04:11,729 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:04:11,729 INFO  [M:0;c4c650e80f58:41613 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:11,729 INFO  [M:0;c4c650e80f58:41613 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:11,729 DEBUG [M:0;c4c650e80f58:41613 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:04:11,729 DEBUG [M:0;c4c650e80f58:41613 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:04:11,729 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513757650 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513757650,5,FailOnTimeoutGroup]
2024-11-13T16:04:11,729 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:04:11,729 INFO  [M:0;c4c650e80f58:41613 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:11,729 INFO  [M:0;c4c650e80f58:41613 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:11,729 DEBUG [M:0;c4c650e80f58:41613 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:04:11,730 INFO  [M:0;c4c650e80f58:41613 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:04:11,730 INFO  [M:0;c4c650e80f58:41613 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:11,730 INFO  [M:0;c4c650e80f58:41613 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:04:11,730 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:04:11,731 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513757651 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513757651,5,FailOnTimeoutGroup]
2024-11-13T16:04:11,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:04:11,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:11,733 DEBUG [M:0;c4c650e80f58:41613 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false
2024-11-13T16:04:11,733 DEBUG [M:0;c4c650e80f58:41613 {}] master.ActiveMasterManager(353): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master
2024-11-13T16:04:11,734 INFO  [M:0;c4c650e80f58:41613 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/.lastflushedseqids
2024-11-13T16:04:11,746 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741854_1030 (size=130)
2024-11-13T16:04:11,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741854_1030 (size=130)
2024-11-13T16:04:11,826 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:11,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:11,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40133-0x100a606b5f90001, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:11,828 INFO  [RS:0;c4c650e80f58:40133 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:11,828 INFO  [RS:0;c4c650e80f58:40133 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,40133,1731513756662; zookeeper connection closed.
2024-11-13T16:04:11,828 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4308aea4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4308aea4
2024-11-13T16:04:11,829 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:04:12,147 INFO  [M:0;c4c650e80f58:41613 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:04:12,148 INFO  [M:0;c4c650e80f58:41613 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:04:12,148 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:04:12,148 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:12,148 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:12,148 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:04:12,148 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:12,148 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB
2024-11-13T16:04:12,167 DEBUG [M:0;c4c650e80f58:41613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/111201cd24f148d2beb2094e3316d229 is 82, key is hbase:meta,,1/info:regioninfo/1731513758508/Put/seqid=0
2024-11-13T16:04:12,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741855_1031 (size=5672)
2024-11-13T16:04:12,173 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741855_1031 (size=5672)
2024-11-13T16:04:12,174 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/111201cd24f148d2beb2094e3316d229
2024-11-13T16:04:12,197 DEBUG [M:0;c4c650e80f58:41613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e87069bd22d9423eb711d4ab615b5715 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731513759767/Put/seqid=0
2024-11-13T16:04:12,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741856_1032 (size=6247)
2024-11-13T16:04:12,204 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741856_1032 (size=6247)
2024-11-13T16:04:12,204 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e87069bd22d9423eb711d4ab615b5715
2024-11-13T16:04:12,211 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e87069bd22d9423eb711d4ab615b5715
2024-11-13T16:04:12,229 DEBUG [M:0;c4c650e80f58:41613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/108dda5cd4b4463785183d4c043b06c9 is 69, key is c4c650e80f58,40133,1731513756662/rs:state/1731513757712/Put/seqid=0
2024-11-13T16:04:12,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741857_1033 (size=5156)
2024-11-13T16:04:12,238 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741857_1033 (size=5156)
2024-11-13T16:04:12,239 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/108dda5cd4b4463785183d4c043b06c9
2024-11-13T16:04:12,264 DEBUG [M:0;c4c650e80f58:41613 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5ad6bb2c2fc043bb8c1c54483b4e8d84 is 52, key is load_balancer_on/state:d/1731513758864/Put/seqid=0
2024-11-13T16:04:12,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741858_1034 (size=5056)
2024-11-13T16:04:12,271 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741858_1034 (size=5056)
2024-11-13T16:04:12,271 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5ad6bb2c2fc043bb8c1c54483b4e8d84
2024-11-13T16:04:12,280 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/111201cd24f148d2beb2094e3316d229 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/111201cd24f148d2beb2094e3316d229
2024-11-13T16:04:12,287 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/111201cd24f148d2beb2094e3316d229, entries=8, sequenceid=59, filesize=5.5 K
2024-11-13T16:04:12,288 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e87069bd22d9423eb711d4ab615b5715 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e87069bd22d9423eb711d4ab615b5715
2024-11-13T16:04:12,295 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e87069bd22d9423eb711d4ab615b5715
2024-11-13T16:04:12,295 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e87069bd22d9423eb711d4ab615b5715, entries=6, sequenceid=59, filesize=6.1 K
2024-11-13T16:04:12,296 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/108dda5cd4b4463785183d4c043b06c9 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/108dda5cd4b4463785183d4c043b06c9
2024-11-13T16:04:12,305 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/108dda5cd4b4463785183d4c043b06c9, entries=1, sequenceid=59, filesize=5.0 K
2024-11-13T16:04:12,306 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/5ad6bb2c2fc043bb8c1c54483b4e8d84 as hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5ad6bb2c2fc043bb8c1c54483b4e8d84
2024-11-13T16:04:12,314 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/5ad6bb2c2fc043bb8c1c54483b4e8d84, entries=1, sequenceid=59, filesize=4.9 K
2024-11-13T16:04:12,316 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false
2024-11-13T16:04:12,318 INFO  [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:12,319 DEBUG [M:0;c4c650e80f58:41613 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513852148Disabling compacts and flushes for region at 1731513852148Disabling writes for close at 1731513852148Obtaining lock to block concurrent updates at 1731513852148Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731513852148Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731513852149 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731513852150 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731513852150Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731513852166 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731513852166Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731513852180 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731513852196 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731513852197 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731513852211 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731513852229 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731513852229Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731513852246 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731513852263 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731513852263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3257eea3: reopening flushed file at 1731513852279 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27d57984: reopening flushed file at 1731513852287 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c41d2c5: reopening flushed file at 1731513852295 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16e25ee: reopening flushed file at 1731513852305 (+10 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 167ms, sequenceid=59, compaction requested=false at 1731513852316 (+11 ms)Writing region close event to WAL at 1731513852318 (+2 ms)Closed at 1731513852318
2024-11-13T16:04:12,319 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:12,320 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:12,320 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:12,320 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:12,320 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:12,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43103 is added to blk_1073741830_1006 (size=27973)
2024-11-13T16:04:12,324 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38019 is added to blk_1073741830_1006 (size=27973)
2024-11-13T16:04:12,328 INFO  [M:0;c4c650e80f58:41613 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:04:12,328 INFO  [M:0;c4c650e80f58:41613 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41613
2024-11-13T16:04:12,329 INFO  [M:0;c4c650e80f58:41613 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:12,330 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:12,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:12,431 INFO  [M:0;c4c650e80f58:41613 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:12,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41613-0x100a606b5f90000, quorum=127.0.0.1:52585, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:12,436 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:12,439 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:12,439 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:12,439 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:12,439 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:12,442 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:12,442 WARN  [BP-662927508-172.17.0.3-1731513753042 heartbeating to localhost/127.0.0.1:38585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:12,443 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:12,443 WARN  [BP-662927508-172.17.0.3-1731513753042 heartbeating to localhost/127.0.0.1:38585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-662927508-172.17.0.3-1731513753042 (Datanode Uuid 3ce1265b-b697-4ebb-a6c9-1ada4c677405) service to localhost/127.0.0.1:38585
2024-11-13T16:04:12,444 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data3/current/BP-662927508-172.17.0.3-1731513753042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:12,445 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data4/current/BP-662927508-172.17.0.3-1731513753042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:12,445 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:12,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:12,448 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:12,448 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:12,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:12,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:12,450 WARN  [BP-662927508-172.17.0.3-1731513753042 heartbeating to localhost/127.0.0.1:38585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:12,450 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:12,450 WARN  [BP-662927508-172.17.0.3-1731513753042 heartbeating to localhost/127.0.0.1:38585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-662927508-172.17.0.3-1731513753042 (Datanode Uuid a6428107-5e4b-458a-b0bc-1fbf259c4138) service to localhost/127.0.0.1:38585
2024-11-13T16:04:12,450 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:12,451 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data1/current/BP-662927508-172.17.0.3-1731513753042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:12,451 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/cluster_2fd6ac08-14ae-7668-55c0-27f7bf745099/data/data2/current/BP-662927508-172.17.0.3-1731513753042 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:12,451 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:12,461 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:04:12,462 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:12,462 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:12,462 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:12,462 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:12,473 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:04:12,505 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:04:12,515 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12)
Potentially hanging thread: nioEventLoopGroup-4-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:38585
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Async-Client-Retry-Timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: HMaster-EventLoopGroup-1-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@2a3f5a0d
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253)
	app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46)
	app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38585 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: ForkJoinPool-2-worker-2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: regionserver/c4c650e80f58:0.procedureResultReporter
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75)

Potentially hanging thread: HMaster-EventLoopGroup-1-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38585 from jenkins.hfs.0
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-4-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RpcClient-timer-pool-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598)
	app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.3
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SessionTracker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163)

Potentially hanging thread: master/c4c650e80f58:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: SnapshotHandlerChoreCleaner
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-1-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Monitor thread for TaskMonitor
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: HBase-Metrics2-1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: master/c4c650e80f58:0:becomeActiveMaster-MemStoreChunkPool Statistics
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-4-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-2-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-5-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:38585
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:38585 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-3-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Time-limited test.named-queue-events-pool-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47)
	app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56)
	app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159)
	app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.1
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: SSL Certificates Store Monitor
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.Object.wait(Object.java:338)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: region-location-0
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170)
	java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-3-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: nioEventLoopGroup-5-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: weak-ref-cleaner-strictcontextstorage
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
	java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176)
	app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: sync.2
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: Timer for 'HBase' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: sync.4
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426)
	app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38585
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=404 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=123 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=11562 (was 11854)
2024-11-13T16:04:12,523 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=404, MaxFileDescriptor=1048576, SystemLoadAverage=123, ProcessCount=11, AvailableMemoryMB=11561
2024-11-13T16:04:12,523 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:04:12,524 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.log.dir so I do NOT create it in target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0
2024-11-13T16:04:12,524 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e9309fec-3f10-1969-824d-3467175837d9/hadoop.tmp.dir so I do NOT create it in target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0
2024-11-13T16:04:12,524 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a, deleteOnExit=true
2024-11-13T16:04:12,524 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/test.cache.data in system properties and HBase conf
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:04:12,525 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:04:12,525 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:04:12,526 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:04:12,527 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:04:12,547 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:04:12,627 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:12,632 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:12,633 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:12,633 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:12,634 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:12,634 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:12,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c69b616{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:12,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5a9b9566{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:12,755 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b50f743{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/java.io.tmpdir/jetty-localhost-44685-hadoop-hdfs-3_4_1-tests_jar-_-any-12666124446495832344/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:04:12,755 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30a1c2a3{HTTP/1.1, (http/1.1)}{localhost:44685}
2024-11-13T16:04:12,756 INFO  [Time-limited test {}] server.Server(415): Started @101641ms
2024-11-13T16:04:12,770 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:04:12,844 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:12,849 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:12,850 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:12,850 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:12,850 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:04:12,850 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9809f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:12,851 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52826651{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:12,972 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@746c6bd5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/java.io.tmpdir/jetty-localhost-36247-hadoop-hdfs-3_4_1-tests_jar-_-any-16548038526984399353/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:12,973 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7517d9e5{HTTP/1.1, (http/1.1)}{localhost:36247}
2024-11-13T16:04:12,973 INFO  [Time-limited test {}] server.Server(415): Started @101859ms
2024-11-13T16:04:12,975 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:13,016 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:13,021 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:13,022 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:13,023 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:13,023 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:13,023 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11d14b2b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:13,024 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@21ea04e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:13,093 WARN  [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data2/current/BP-875556364-172.17.0.3-1731513852567/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:13,093 WARN  [Thread-437 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data1/current/BP-875556364-172.17.0.3-1731513852567/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:13,114 WARN  [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:13,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa0769bcbb25c7fca with lease ID 0x741761bd79250a28: Processing first storage report for DS-2ed615cf-9491-4b6c-8e9f-0849b2a50c86 from datanode DatanodeRegistration(127.0.0.1:36679, datanodeUuid=6e9242dd-b3f6-41a0-8fb6-971aa5e3f4b7, infoPort=36491, infoSecurePort=0, ipcPort=33439, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567)
2024-11-13T16:04:13,118 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa0769bcbb25c7fca with lease ID 0x741761bd79250a28: from storage DS-2ed615cf-9491-4b6c-8e9f-0849b2a50c86 node DatanodeRegistration(127.0.0.1:36679, datanodeUuid=6e9242dd-b3f6-41a0-8fb6-971aa5e3f4b7, infoPort=36491, infoSecurePort=0, ipcPort=33439, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:04:13,118 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa0769bcbb25c7fca with lease ID 0x741761bd79250a28: Processing first storage report for DS-c7523ef4-97b4-4805-872f-a07357383e00 from datanode DatanodeRegistration(127.0.0.1:36679, datanodeUuid=6e9242dd-b3f6-41a0-8fb6-971aa5e3f4b7, infoPort=36491, infoSecurePort=0, ipcPort=33439, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567)
2024-11-13T16:04:13,118 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa0769bcbb25c7fca with lease ID 0x741761bd79250a28: from storage DS-c7523ef4-97b4-4805-872f-a07357383e00 node DatanodeRegistration(127.0.0.1:36679, datanodeUuid=6e9242dd-b3f6-41a0-8fb6-971aa5e3f4b7, infoPort=36491, infoSecurePort=0, ipcPort=33439, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:13,145 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@626a143d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/java.io.tmpdir/jetty-localhost-45905-hadoop-hdfs-3_4_1-tests_jar-_-any-11124572487954726387/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:13,146 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@aab268d{HTTP/1.1, (http/1.1)}{localhost:45905}
2024-11-13T16:04:13,146 INFO  [Time-limited test {}] server.Server(415): Started @102031ms
2024-11-13T16:04:13,147 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:13,273 WARN  [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data4/current/BP-875556364-172.17.0.3-1731513852567/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:13,273 WARN  [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data3/current/BP-875556364-172.17.0.3-1731513852567/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:13,294 WARN  [Thread-452 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:13,297 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xae7725d2b9344768 with lease ID 0x741761bd79250a29: Processing first storage report for DS-6befb814-a9a5-49bf-8d50-84453cfe3a92 from datanode DatanodeRegistration(127.0.0.1:33963, datanodeUuid=c87e90d6-5133-419d-9971-d19ad0853a0c, infoPort=45285, infoSecurePort=0, ipcPort=46305, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567)
2024-11-13T16:04:13,298 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae7725d2b9344768 with lease ID 0x741761bd79250a29: from storage DS-6befb814-a9a5-49bf-8d50-84453cfe3a92 node DatanodeRegistration(127.0.0.1:33963, datanodeUuid=c87e90d6-5133-419d-9971-d19ad0853a0c, infoPort=45285, infoSecurePort=0, ipcPort=46305, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:04:13,298 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xae7725d2b9344768 with lease ID 0x741761bd79250a29: Processing first storage report for DS-ad5d66e0-18ab-4f4c-a714-43bcf7917faf from datanode DatanodeRegistration(127.0.0.1:33963, datanodeUuid=c87e90d6-5133-419d-9971-d19ad0853a0c, infoPort=45285, infoSecurePort=0, ipcPort=46305, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567)
2024-11-13T16:04:13,298 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xae7725d2b9344768 with lease ID 0x741761bd79250a29: from storage DS-ad5d66e0-18ab-4f4c-a714-43bcf7917faf node DatanodeRegistration(127.0.0.1:33963, datanodeUuid=c87e90d6-5133-419d-9971-d19ad0853a0c, infoPort=45285, infoSecurePort=0, ipcPort=46305, storageInfo=lv=-57;cid=testClusterID;nsid=1302845585;c=1731513852567), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:13,387 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0
2024-11-13T16:04:13,399 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/zookeeper_0, clientPort=58372, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:04:13,401 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58372
2024-11-13T16:04:13,401 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,403 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:04:13,418 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:04:13,420 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9 with version=8
2024-11-13T16:04:13,420 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:04:13,424 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:04:13,425 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:04:13,426 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39847
2024-11-13T16:04:13,427 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39847 connecting to ZooKeeper ensemble=127.0.0.1:58372
2024-11-13T16:04:13,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398470x0, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:04:13,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39847-0x100a60835ed0000 connected
2024-11-13T16:04:13,506 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,508 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:13,513 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9, hbase.cluster.distributed=false
2024-11-13T16:04:13,516 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:04:13,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39847
2024-11-13T16:04:13,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39847
2024-11-13T16:04:13,522 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39847
2024-11-13T16:04:13,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39847
2024-11-13T16:04:13,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39847
2024-11-13T16:04:13,556 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:04:13,556 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:04:13,557 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:04:13,558 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42811
2024-11-13T16:04:13,559 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42811 connecting to ZooKeeper ensemble=127.0.0.1:58372
2024-11-13T16:04:13,560 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,563 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428110x0, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:04:13,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42811-0x100a60835ed0001 connected
2024-11-13T16:04:13,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:13,576 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:04:13,581 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:04:13,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:04:13,586 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:04:13,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42811
2024-11-13T16:04:13,593 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42811
2024-11-13T16:04:13,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42811
2024-11-13T16:04:13,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42811
2024-11-13T16:04:13,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42811
2024-11-13T16:04:13,617 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:39847
2024-11-13T16:04:13,618 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:13,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:13,621 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:04:13,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,624 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:04:13,624 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,39847,1731513853423 from backup master directory
2024-11-13T16:04:13,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:13,627 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:04:13,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:13,627 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,636 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/hbase.id] with ID: b92b3b86-c1f1-4095-85fc-90a1413109b0
2024-11-13T16:04:13,636 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/.tmp/hbase.id
2024-11-13T16:04:13,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:04:13,646 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:04:13,647 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/.tmp/hbase.id]:[hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/hbase.id]
2024-11-13T16:04:13,664 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:13,664 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:04:13,667 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms.
2024-11-13T16:04:13,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,689 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:04:13,690 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:04:13,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:04:13,692 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:04:13,693 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:13,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:04:13,709 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:04:13,711 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store
2024-11-13T16:04:13,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:04:13,720 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:04:13,721 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:13,722 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:04:13,722 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:13,722 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:13,722 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:04:13,722 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:13,722 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:13,722 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513853722Disabling compacts and flushes for region at 1731513853722Disabling writes for close at 1731513853722Writing region close event to WAL at 1731513853722Closed at 1731513853722
2024-11-13T16:04:13,723 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/.initializing
2024-11-13T16:04:13,724 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/WALs/c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,727 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C39847%2C1731513853423, suffix=, logDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/WALs/c4c650e80f58,39847,1731513853423, archiveDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/oldWALs, maxLogs=10
2024-11-13T16:04:13,728 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C39847%2C1731513853423.1731513853728
2024-11-13T16:04:13,735 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/WALs/c4c650e80f58,39847,1731513853423/c4c650e80f58%2C39847%2C1731513853423.1731513853728
2024-11-13T16:04:13,743 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:36491:36491)]
2024-11-13T16:04:13,746 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:04:13,747 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:13,747 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,747 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,750 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:13,752 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:04:13,752 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,753 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:13,753 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:13,755 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:04:13,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,756 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:13,756 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:13,758 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:04:13,758 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,759 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:13,759 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:13,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:04:13,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,761 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:13,761 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,762 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,763 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,765 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,765 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,765 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:04:13,767 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:13,770 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:04:13,771 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868740, jitterRate=0.10466116666793823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:04:13,772 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513853747Initializing all the Stores at 1731513853749 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513853749Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513853749Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513853749Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513853749Cleaning up temporary data from old regions at 1731513853765 (+16 ms)Region opened successfully at 1731513853772 (+7 ms)
2024-11-13T16:04:13,772 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:04:13,776 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414fba34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:04:13,777 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:04:13,778 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:04:13,778 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:04:13,778 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:04:13,779 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:04:13,779 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:04:13,779 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:04:13,782 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:04:13,783 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:04:13,784 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:04:13,785 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:04:13,785 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:04:13,787 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:04:13,787 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:04:13,788 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:04:13,790 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:04:13,791 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:04:13,793 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:04:13,795 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:04:13,797 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:04:13,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:13,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:13,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,799 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,39847,1731513853423, sessionid=0x100a60835ed0000, setting cluster-up flag (Was=false)
2024-11-13T16:04:13,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,804 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,810 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:04:13,811 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:13,820 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:04:13,821 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,39847,1731513853423
2024-11-13T16:04:13,823 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:04:13,825 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:13,826 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:04:13,826 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:04:13,826 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,39847,1731513853423
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:04:13,827 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:13,827 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:13,827 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:13,828 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:13,828 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:04:13,828 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,828 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:04:13,828 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,828 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731513883828
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:04:13,829 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,830 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:13,830 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:04:13,830 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:04:13,830 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:04:13,830 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:04:13,831 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,831 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:04:13,834 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:04:13,834 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:04:13,835 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513853835,5,FailOnTimeoutGroup]
2024-11-13T16:04:13,835 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513853835,5,FailOnTimeoutGroup]
2024-11-13T16:04:13,835 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,835 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:04:13,835 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,835 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:04:13,844 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:04:13,846 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:04:13,846 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9
2024-11-13T16:04:13,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:04:13,855 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:04:13,856 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:13,858 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:04:13,859 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:04:13,860 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,860 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:13,860 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:04:13,862 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:04:13,862 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,862 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:13,863 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:04:13,864 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:04:13,864 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,865 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:13,865 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:04:13,866 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:04:13,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:13,867 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:13,867 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:04:13,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740
2024-11-13T16:04:13,868 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740
2024-11-13T16:04:13,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:04:13,870 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:04:13,870 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:04:13,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:04:13,875 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:04:13,876 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=832566, jitterRate=0.05866245925426483}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:04:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513853856Initializing all the Stores at 1731513853857 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513853858 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513853858Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513853858Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513853858Cleaning up temporary data from old regions at 1731513853870 (+12 ms)Region opened successfully at 1731513853878 (+8 ms)
2024-11-13T16:04:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:04:13,878 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:04:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:04:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:04:13,878 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:04:13,879 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:13,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513853878Disabling compacts and flushes for region at 1731513853878Disabling writes for close at 1731513853878Writing region close event to WAL at 1731513853878Closed at 1731513853879 (+1 ms)
2024-11-13T16:04:13,881 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:13,881 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:04:13,881 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:04:13,883 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:04:13,885 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:04:13,904 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(746): ClusterId : b92b3b86-c1f1-4095-85fc-90a1413109b0
2024-11-13T16:04:13,904 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:04:13,907 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:04:13,907 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:04:13,910 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:04:13,911 DEBUG [RS:0;c4c650e80f58:42811 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f00d9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:04:13,926 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:42811
2024-11-13T16:04:13,926 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:04:13,926 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:04:13,926 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:04:13,927 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,39847,1731513853423 with port=42811, startcode=1731513853556
2024-11-13T16:04:13,927 DEBUG [RS:0;c4c650e80f58:42811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:04:13,931 INFO  [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52879, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:04:13,932 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39847 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,932 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39847 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,936 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9
2024-11-13T16:04:13,936 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37687
2024-11-13T16:04:13,936 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:04:13,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:13,944 DEBUG [RS:0;c4c650e80f58:42811 {}] zookeeper.ZKUtil(111): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,944 WARN  [RS:0;c4c650e80f58:42811 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:04:13,944 INFO  [RS:0;c4c650e80f58:42811 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:13,945 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/WALs/c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,945 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,42811,1731513853556]
2024-11-13T16:04:13,952 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:04:13,955 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:04:13,956 INFO  [RS:0;c4c650e80f58:42811 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:04:13,956 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,957 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:04:13,958 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:04:13,958 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,958 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,958 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,958 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:13,959 DEBUG [RS:0;c4c650e80f58:42811 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:13,962 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,962 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,962 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,962 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,962 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,963 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,42811,1731513853556-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:04:13,980 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:04:13,980 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,42811,1731513853556-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,981 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,981 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.Replication(171): c4c650e80f58,42811,1731513853556 started
2024-11-13T16:04:13,997 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:13,997 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,42811,1731513853556, RpcServer on c4c650e80f58/172.17.0.3:42811, sessionid=0x100a60835ed0001
2024-11-13T16:04:13,997 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:04:13,997 DEBUG [RS:0;c4c650e80f58:42811 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,997 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,42811,1731513853556'
2024-11-13T16:04:13,997 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:04:13,998 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,42811,1731513853556
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,42811,1731513853556'
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:04:13,999 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:04:14,000 DEBUG [RS:0;c4c650e80f58:42811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:04:14,000 INFO  [RS:0;c4c650e80f58:42811 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:04:14,000 INFO  [RS:0;c4c650e80f58:42811 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:04:14,035 WARN  [c4c650e80f58:39847 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:04:14,103 INFO  [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C42811%2C1731513853556, suffix=, logDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/WALs/c4c650e80f58,42811,1731513853556, archiveDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/oldWALs, maxLogs=32
2024-11-13T16:04:14,105 INFO  [RS:0;c4c650e80f58:42811 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C42811%2C1731513853556.1731513854105
2024-11-13T16:04:14,120 INFO  [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/WALs/c4c650e80f58,42811,1731513853556/c4c650e80f58%2C42811%2C1731513853556.1731513854105
2024-11-13T16:04:14,131 DEBUG [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:36491:36491)]
2024-11-13T16:04:14,285 DEBUG [c4c650e80f58:39847 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:04:14,286 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,42811,1731513853556
2024-11-13T16:04:14,289 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,42811,1731513853556, state=OPENING
2024-11-13T16:04:14,291 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:04:14,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:14,293 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:14,294 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:04:14,295 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,42811,1731513853556}]
2024-11-13T16:04:14,295 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:14,295 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:14,449 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:04:14,452 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:04:14,457 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:04:14,457 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:14,459 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C42811%2C1731513853556.meta, suffix=.meta, logDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/WALs/c4c650e80f58,42811,1731513853556, archiveDir=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/oldWALs, maxLogs=32
2024-11-13T16:04:14,461 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C42811%2C1731513853556.meta.1731513854461.meta
2024-11-13T16:04:14,468 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/WALs/c4c650e80f58,42811,1731513853556/c4c650e80f58%2C42811%2C1731513853556.meta.1731513854461.meta
2024-11-13T16:04:14,469 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45285:45285),(127.0.0.1/127.0.0.1:36491:36491)]
2024-11-13T16:04:14,471 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:04:14,471 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:04:14,471 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:04:14,472 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:04:14,472 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:04:14,472 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:14,472 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:04:14,472 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:04:14,474 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:04:14,475 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:04:14,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:14,476 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:14,476 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:04:14,477 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:04:14,477 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:14,478 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:14,478 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:04:14,479 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:04:14,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:14,480 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:14,480 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:04:14,481 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:04:14,481 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:14,481 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:14,482 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:04:14,482 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740
2024-11-13T16:04:14,484 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740
2024-11-13T16:04:14,486 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:04:14,486 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:04:14,487 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:04:14,488 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:04:14,489 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=782671, jitterRate=-0.004782810807228088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:04:14,490 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:04:14,491 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513854472Writing region info on filesystem at 1731513854472Initializing all the Stores at 1731513854474 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513854474Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513854474Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513854474Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513854474Cleaning up temporary data from old regions at 1731513854486 (+12 ms)Running coprocessor post-open hooks at 1731513854490 (+4 ms)Region opened successfully at 1731513854491 (+1 ms)
2024-11-13T16:04:14,492 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513854449
2024-11-13T16:04:14,495 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:04:14,495 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:04:14,496 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,42811,1731513853556
2024-11-13T16:04:14,498 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,42811,1731513853556, state=OPEN
2024-11-13T16:04:14,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:04:14,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:04:14,504 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,42811,1731513853556
2024-11-13T16:04:14,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:14,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:14,508 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:04:14,508 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,42811,1731513853556 in 210 msec
2024-11-13T16:04:14,511 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:04:14,511 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 627 msec
2024-11-13T16:04:14,512 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:14,513 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:04:14,514 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:04:14,514 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,42811,1731513853556, seqNum=-1]
2024-11-13T16:04:14,515 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:04:14,517 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32821, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:04:14,525 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 698 msec
2024-11-13T16:04:14,525 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513854525, completionTime=-1
2024-11-13T16:04:14,525 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:04:14,525 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:04:14,527 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:04:14,527 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731513914527
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731513974528
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:39847, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,528 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,531 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.905sec
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:04:14,533 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:04:14,534 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:04:14,534 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:04:14,536 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:04:14,537 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:04:14,537 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39847,1731513853423-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:14,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6469263a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:14,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,39847,-1 for getting cluster id
2024-11-13T16:04:14,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:04:14,607 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b92b3b86-c1f1-4095-85fc-90a1413109b0'
2024-11-13T16:04:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:04:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b92b3b86-c1f1-4095-85fc-90a1413109b0"

2024-11-13T16:04:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7047af98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,39847,-1]
2024-11-13T16:04:14,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:04:14,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,610 INFO  [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:04:14,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@466b5edb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:14,612 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:04:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,42811,1731513853556, seqNum=-1]
2024-11-13T16:04:14,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:04:14,615 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:04:14,617 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,39847,1731513853423
2024-11-13T16:04:14,617 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:14,621 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:04:14,621 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:04:14,621 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:14,621 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:14,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,621 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:04:14,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:04:14,621 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1555122662, stopped=false
2024-11-13T16:04:14,622 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,39847,1731513853423
2024-11-13T16:04:14,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:14,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:14,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:14,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:14,624 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:14,624 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:14,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:14,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,624 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:14,625 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,42811,1731513853556' *****
2024-11-13T16:04:14,625 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:04:14,625 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:04:14,625 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,42811,1731513853556
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:14,625 INFO  [RS:0;c4c650e80f58:42811 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:42811.
2024-11-13T16:04:14,625 DEBUG [RS:0;c4c650e80f58:42811 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:14,626 DEBUG [RS:0;c4c650e80f58:42811 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,626 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:04:14,626 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:04:14,626 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:04:14,626 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:04:14,626 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-13T16:04:14,626 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-13T16:04:14,626 DEBUG [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-13T16:04:14,626 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:04:14,626 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:04:14,626 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:04:14,626 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:04:14,627 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:04:14,627 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-13T16:04:14,646 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/.tmp/ns/30ecdcea1a7b4cc9a2ad922de3464533 is 43, key is default/ns:d/1731513854517/Put/seqid=0
2024-11-13T16:04:14,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741835_1011 (size=5153)
2024-11-13T16:04:14,652 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741835_1011 (size=5153)
2024-11-13T16:04:14,653 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/.tmp/ns/30ecdcea1a7b4cc9a2ad922de3464533
2024-11-13T16:04:14,660 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/.tmp/ns/30ecdcea1a7b4cc9a2ad922de3464533 as hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/ns/30ecdcea1a7b4cc9a2ad922de3464533
2024-11-13T16:04:14,667 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/ns/30ecdcea1a7b4cc9a2ad922de3464533, entries=2, sequenceid=6, filesize=5.0 K
2024-11-13T16:04:14,669 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false
2024-11-13T16:04:14,669 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-13T16:04:14,674 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-13T16:04:14,675 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:14,675 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:14,675 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513854626Running coprocessor pre-close hooks at 1731513854626Disabling compacts and flushes for region at 1731513854626Disabling writes for close at 1731513854626Obtaining lock to block concurrent updates at 1731513854627 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731513854627Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731513854627Flushing stores of hbase:meta,,1.1588230740 at 1731513854628 (+1 ms)Flushing 1588230740/ns: creating writer at 1731513854628Flushing 1588230740/ns: appending metadata at 1731513854645 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731513854645Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c7d9c2d: reopening flushed file at 1731513854659 (+14 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false at 1731513854669 (+10 ms)Writing region close event to WAL at 1731513854670 (+1 ms)Running coprocessor post-close hooks at 1731513854675 (+5 ms)Closed at 1731513854675
2024-11-13T16:04:14,675 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:14,826 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,42811,1731513853556; all regions closed.
2024-11-13T16:04:14,827 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,827 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,827 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,828 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,828 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,830 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741834_1010 (size=1152)
2024-11-13T16:04:14,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741834_1010 (size=1152)
2024-11-13T16:04:14,833 DEBUG [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/oldWALs
2024-11-13T16:04:14,833 INFO  [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C42811%2C1731513853556.meta:.meta(num 1731513854461)
2024-11-13T16:04:14,833 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,834 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,834 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,834 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,834 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:14,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741833_1009 (size=93)
2024-11-13T16:04:14,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741833_1009 (size=93)
2024-11-13T16:04:14,839 DEBUG [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/oldWALs
2024-11-13T16:04:14,839 INFO  [RS:0;c4c650e80f58:42811 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C42811%2C1731513853556:(num 1731513854105)
2024-11-13T16:04:14,839 DEBUG [RS:0;c4c650e80f58:42811 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:14,839 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:14,839 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:14,839 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:14,839 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:14,840 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:14,840 INFO  [RS:0;c4c650e80f58:42811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42811
2024-11-13T16:04:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,42811,1731513853556
2024-11-13T16:04:14,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:14,842 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:14,845 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,42811,1731513853556]
2024-11-13T16:04:14,846 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,42811,1731513853556 already deleted, retry=false
2024-11-13T16:04:14,846 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,42811,1731513853556 expired; onlineServers=0
2024-11-13T16:04:14,846 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,39847,1731513853423' *****
2024-11-13T16:04:14,846 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:04:14,846 INFO  [M:0;c4c650e80f58:39847 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:14,846 INFO  [M:0;c4c650e80f58:39847 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:14,846 DEBUG [M:0;c4c650e80f58:39847 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:04:14,847 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:04:14,847 DEBUG [M:0;c4c650e80f58:39847 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:04:14,847 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513853835 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513853835,5,FailOnTimeoutGroup]
2024-11-13T16:04:14,847 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513853835 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513853835,5,FailOnTimeoutGroup]
2024-11-13T16:04:14,847 INFO  [M:0;c4c650e80f58:39847 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:14,847 INFO  [M:0;c4c650e80f58:39847 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:14,847 DEBUG [M:0;c4c650e80f58:39847 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:04:14,847 INFO  [M:0;c4c650e80f58:39847 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:04:14,847 INFO  [M:0;c4c650e80f58:39847 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:14,848 INFO  [M:0;c4c650e80f58:39847 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:04:14,848 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:04:14,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:04:14,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:14,849 DEBUG [M:0;c4c650e80f58:39847 {}] zookeeper.ZKUtil(347): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:04:14,849 WARN  [M:0;c4c650e80f58:39847 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:04:14,850 INFO  [M:0;c4c650e80f58:39847 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/.lastflushedseqids
2024-11-13T16:04:14,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741836_1012 (size=99)
2024-11-13T16:04:14,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741836_1012 (size=99)
2024-11-13T16:04:14,856 INFO  [M:0;c4c650e80f58:39847 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:04:14,857 INFO  [M:0;c4c650e80f58:39847 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:04:14,857 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:04:14,857 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:14,857 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:14,857 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:04:14,857 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:14,857 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-13T16:04:14,876 DEBUG [M:0;c4c650e80f58:39847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f79afd6c20a8465b8744e3aa5d10214e is 82, key is hbase:meta,,1/info:regioninfo/1731513854496/Put/seqid=0
2024-11-13T16:04:14,881 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741837_1013 (size=5672)
2024-11-13T16:04:14,882 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741837_1013 (size=5672)
2024-11-13T16:04:14,882 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f79afd6c20a8465b8744e3aa5d10214e
2024-11-13T16:04:14,906 DEBUG [M:0;c4c650e80f58:39847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0edee247f3b4440e803a97ae6827dbcb is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731513854523/Put/seqid=0
2024-11-13T16:04:14,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741838_1014 (size=5275)
2024-11-13T16:04:14,912 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741838_1014 (size=5275)
2024-11-13T16:04:14,912 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0edee247f3b4440e803a97ae6827dbcb
2024-11-13T16:04:14,935 DEBUG [M:0;c4c650e80f58:39847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8630a9fd889b44e1b774df5d41ae67a1 is 69, key is c4c650e80f58,42811,1731513853556/rs:state/1731513853932/Put/seqid=0
2024-11-13T16:04:14,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741839_1015 (size=5156)
2024-11-13T16:04:14,941 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741839_1015 (size=5156)
2024-11-13T16:04:14,942 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8630a9fd889b44e1b774df5d41ae67a1
2024-11-13T16:04:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:14,945 INFO  [RS:0;c4c650e80f58:42811 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:14,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42811-0x100a60835ed0001, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:14,945 INFO  [RS:0;c4c650e80f58:42811 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,42811,1731513853556; zookeeper connection closed.
2024-11-13T16:04:14,945 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7cf21c7f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7cf21c7f
2024-11-13T16:04:14,945 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:04:14,966 DEBUG [M:0;c4c650e80f58:39847 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d5d46f9416941dc9b07e3577a5d71dc is 52, key is load_balancer_on/state:d/1731513854619/Put/seqid=0
2024-11-13T16:04:14,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741840_1016 (size=5056)
2024-11-13T16:04:14,972 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741840_1016 (size=5056)
2024-11-13T16:04:14,972 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d5d46f9416941dc9b07e3577a5d71dc
2024-11-13T16:04:14,979 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f79afd6c20a8465b8744e3aa5d10214e as hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f79afd6c20a8465b8744e3aa5d10214e
2024-11-13T16:04:14,985 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f79afd6c20a8465b8744e3aa5d10214e, entries=8, sequenceid=29, filesize=5.5 K
2024-11-13T16:04:14,987 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0edee247f3b4440e803a97ae6827dbcb as hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0edee247f3b4440e803a97ae6827dbcb
2024-11-13T16:04:14,992 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0edee247f3b4440e803a97ae6827dbcb, entries=3, sequenceid=29, filesize=5.2 K
2024-11-13T16:04:14,994 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8630a9fd889b44e1b774df5d41ae67a1 as hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8630a9fd889b44e1b774df5d41ae67a1
2024-11-13T16:04:15,000 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8630a9fd889b44e1b774df5d41ae67a1, entries=1, sequenceid=29, filesize=5.0 K
2024-11-13T16:04:15,001 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d5d46f9416941dc9b07e3577a5d71dc as hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1d5d46f9416941dc9b07e3577a5d71dc
2024-11-13T16:04:15,007 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37687/user/jenkins/test-data/4310df88-95ed-f70c-f01c-4a853097e9c9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1d5d46f9416941dc9b07e3577a5d71dc, entries=1, sequenceid=29, filesize=4.9 K
2024-11-13T16:04:15,009 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 152ms, sequenceid=29, compaction requested=false
2024-11-13T16:04:15,011 INFO  [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:15,011 DEBUG [M:0;c4c650e80f58:39847 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513854857Disabling compacts and flushes for region at 1731513854857Disabling writes for close at 1731513854857Obtaining lock to block concurrent updates at 1731513854857Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731513854857Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731513854857Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731513854859 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731513854859Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731513854875 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731513854875Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731513854888 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731513854905 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731513854905Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731513854919 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731513854935 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731513854935Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731513854949 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731513854965 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731513854965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46005eb9: reopening flushed file at 1731513854978 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40ae915b: reopening flushed file at 1731513854986 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3979920e: reopening flushed file at 1731513854993 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60ee412: reopening flushed file at 1731513855000 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 152ms, sequenceid=29, compaction requested=false at 1731513855009 (+9 ms)Writing region close event to WAL at 1731513855010 (+1 ms)Closed at 1731513855010
2024-11-13T16:04:15,011 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:15,011 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:15,011 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:15,012 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:15,012 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:15,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36679 is added to blk_1073741830_1006 (size=10311)
2024-11-13T16:04:15,015 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33963 is added to blk_1073741830_1006 (size=10311)
2024-11-13T16:04:15,015 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:15,015 INFO  [M:0;c4c650e80f58:39847 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:04:15,016 INFO  [M:0;c4c650e80f58:39847 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39847
2024-11-13T16:04:15,016 INFO  [M:0;c4c650e80f58:39847 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:15,118 INFO  [M:0;c4c650e80f58:39847 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:15,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:15,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39847-0x100a60835ed0000, quorum=127.0.0.1:58372, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:15,121 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@626a143d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:15,121 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@aab268d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:15,122 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:15,122 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@21ea04e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:15,122 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11d14b2b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:15,123 WARN  [BP-875556364-172.17.0.3-1731513852567 heartbeating to localhost/127.0.0.1:37687 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:15,123 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:15,123 WARN  [BP-875556364-172.17.0.3-1731513852567 heartbeating to localhost/127.0.0.1:37687 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-875556364-172.17.0.3-1731513852567 (Datanode Uuid c87e90d6-5133-419d-9971-d19ad0853a0c) service to localhost/127.0.0.1:37687
2024-11-13T16:04:15,123 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:15,124 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data3/current/BP-875556364-172.17.0.3-1731513852567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:15,124 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data4/current/BP-875556364-172.17.0.3-1731513852567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:15,124 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:15,127 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@746c6bd5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:15,127 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7517d9e5{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:15,127 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:15,127 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52826651{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:15,127 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9809f98{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:15,129 WARN  [BP-875556364-172.17.0.3-1731513852567 heartbeating to localhost/127.0.0.1:37687 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:15,129 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:15,130 WARN  [BP-875556364-172.17.0.3-1731513852567 heartbeating to localhost/127.0.0.1:37687 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-875556364-172.17.0.3-1731513852567 (Datanode Uuid 6e9242dd-b3f6-41a0-8fb6-971aa5e3f4b7) service to localhost/127.0.0.1:37687
2024-11-13T16:04:15,130 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:15,130 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data1/current/BP-875556364-172.17.0.3-1731513852567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:15,131 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/cluster_e889a2eb-4378-744c-ebf2-bb29cd9cc54a/data/data2/current/BP-875556364-172.17.0.3-1731513852567 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:15,131 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:15,137 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b50f743{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:04:15,138 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30a1c2a3{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:15,138 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:15,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5a9b9566{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:15,139 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c69b616{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:15,145 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.log.dir so I do NOT create it in target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/de2134c2-10c5-e672-04d4-fe22aca909e0/hadoop.tmp.dir so I do NOT create it in target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233, deleteOnExit=true
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/test.cache.data in system properties and HBase conf
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:04:15,161 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:04:15,162 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:04:15,162 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:04:15,163 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:04:15,164 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:04:15,181 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:04:15,261 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:15,266 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:15,270 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:15,270 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:15,270 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:04:15,271 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:15,271 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67cf8368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:15,271 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6de86657{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:15,387 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43ade8e8{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-33247-hadoop-hdfs-3_4_1-tests_jar-_-any-15795342473608835424/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:04:15,388 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ac253d{HTTP/1.1, (http/1.1)}{localhost:33247}
2024-11-13T16:04:15,388 INFO  [Time-limited test {}] server.Server(415): Started @104274ms
2024-11-13T16:04:15,402 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:04:15,472 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:15,476 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:15,476 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:15,477 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:15,477 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:15,477 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e195dbd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:15,478 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2693ea96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:15,594 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1827207e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-34295-hadoop-hdfs-3_4_1-tests_jar-_-any-1584011669229871763/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:15,594 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a18c5e6{HTTP/1.1, (http/1.1)}{localhost:34295}
2024-11-13T16:04:15,594 INFO  [Time-limited test {}] server.Server(415): Started @104480ms
2024-11-13T16:04:15,596 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:15,631 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:15,634 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:15,635 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:15,635 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:15,635 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:15,636 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63c2c4fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:15,637 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45dc61f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:15,736 WARN  [Thread-656 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data1/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:15,736 WARN  [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data2/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:15,761 WARN  [Thread-635 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:15,765 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf68c280d05eff940 with lease ID 0x7b55e8a9c370ad3f: Processing first storage report for DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3 from datanode DatanodeRegistration(127.0.0.1:45619, datanodeUuid=ec6664ce-658a-4dd3-b473-2ab8d4387437, infoPort=46583, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:15,765 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf68c280d05eff940 with lease ID 0x7b55e8a9c370ad3f: from storage DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3 node DatanodeRegistration(127.0.0.1:45619, datanodeUuid=ec6664ce-658a-4dd3-b473-2ab8d4387437, infoPort=46583, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:15,765 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf68c280d05eff940 with lease ID 0x7b55e8a9c370ad3f: Processing first storage report for DS-dc048eb1-70e9-44ff-9b0d-8270604e1bb5 from datanode DatanodeRegistration(127.0.0.1:45619, datanodeUuid=ec6664ce-658a-4dd3-b473-2ab8d4387437, infoPort=46583, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:15,766 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf68c280d05eff940 with lease ID 0x7b55e8a9c370ad3f: from storage DS-dc048eb1-70e9-44ff-9b0d-8270604e1bb5 node DatanodeRegistration(127.0.0.1:45619, datanodeUuid=ec6664ce-658a-4dd3-b473-2ab8d4387437, infoPort=46583, infoSecurePort=0, ipcPort=37367, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:15,773 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b2e9f0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-42521-hadoop-hdfs-3_4_1-tests_jar-_-any-12328487275095684555/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:15,773 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2c64d82b{HTTP/1.1, (http/1.1)}{localhost:42521}
2024-11-13T16:04:15,773 INFO  [Time-limited test {}] server.Server(415): Started @104659ms
2024-11-13T16:04:15,775 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:15,897 WARN  [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:15,897 WARN  [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:15,915 WARN  [Thread-671 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:15,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x897625a276824c35 with lease ID 0x7b55e8a9c370ad40: Processing first storage report for DS-31719626-8e9c-4b40-b0e8-d9156c01922a from datanode DatanodeRegistration(127.0.0.1:46019, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=33141, infoSecurePort=0, ipcPort=39341, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:15,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x897625a276824c35 with lease ID 0x7b55e8a9c370ad40: from storage DS-31719626-8e9c-4b40-b0e8-d9156c01922a node DatanodeRegistration(127.0.0.1:46019, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=33141, infoSecurePort=0, ipcPort=39341, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:15,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x897625a276824c35 with lease ID 0x7b55e8a9c370ad40: Processing first storage report for DS-29d37a25-747e-405c-bdeb-2f855b4990ff from datanode DatanodeRegistration(127.0.0.1:46019, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=33141, infoSecurePort=0, ipcPort=39341, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:15,918 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x897625a276824c35 with lease ID 0x7b55e8a9c370ad40: from storage DS-29d37a25-747e-405c-bdeb-2f855b4990ff node DatanodeRegistration(127.0.0.1:46019, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=33141, infoSecurePort=0, ipcPort=39341, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:15,963 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:16,008 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285
2024-11-13T16:04:16,011 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/zookeeper_0, clientPort=49300, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:04:16,012 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49300
2024-11-13T16:04:16,012 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,014 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:04:16,025 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:04:16,026 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b with version=8
2024-11-13T16:04:16,026 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:04:16,029 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:04:16,030 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40685
2024-11-13T16:04:16,031 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40685 connecting to ZooKeeper ensemble=127.0.0.1:49300
2024-11-13T16:04:16,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:406850x0, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:04:16,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40685-0x100a608402a0000 connected
2024-11-13T16:04:16,055 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,057 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:16,060 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b, hbase.cluster.distributed=false
2024-11-13T16:04:16,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:04:16,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40685
2024-11-13T16:04:16,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40685
2024-11-13T16:04:16,062 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40685
2024-11-13T16:04:16,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40685
2024-11-13T16:04:16,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40685
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:04:16,080 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:04:16,081 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41961
2024-11-13T16:04:16,083 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41961 connecting to ZooKeeper ensemble=127.0.0.1:49300
2024-11-13T16:04:16,083 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,086 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419610x0, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:04:16,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419610x0, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:16,091 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41961-0x100a608402a0001 connected
2024-11-13T16:04:16,091 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:04:16,092 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:04:16,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:04:16,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:04:16,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41961
2024-11-13T16:04:16,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41961
2024-11-13T16:04:16,094 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41961
2024-11-13T16:04:16,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41961
2024-11-13T16:04:16,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41961
2024-11-13T16:04:16,108 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:40685
2024-11-13T16:04:16,109 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:16,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:16,111 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:04:16,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,114 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:04:16,114 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,40685,1731513856028 from backup master directory
2024-11-13T16:04:16,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:16,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:04:16,118 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:04:16,118 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,123 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/hbase.id] with ID: a7ffe9c4-9f97-4bee-a1c9-e04ee1554e10
2024-11-13T16:04:16,123 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/.tmp/hbase.id
2024-11-13T16:04:16,130 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:04:16,131 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:04:16,131 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/.tmp/hbase.id]:[hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/hbase.id]
2024-11-13T16:04:16,146 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:16,146 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:04:16,148 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms.
2024-11-13T16:04:16,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:04:16,159 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:04:16,160 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:04:16,161 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:04:16,162 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:16,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:04:16,171 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:04:16,172 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store
2024-11-13T16:04:16,180 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:04:16,181 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:04:16,181 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:16,181 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:04:16,181 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:16,182 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:16,182 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:04:16,182 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:16,182 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:16,182 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513856181Disabling compacts and flushes for region at 1731513856181Disabling writes for close at 1731513856182 (+1 ms)Writing region close event to WAL at 1731513856182Closed at 1731513856182
2024-11-13T16:04:16,183 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/.initializing
2024-11-13T16:04:16,183 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,186 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C40685%2C1731513856028, suffix=, logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028, archiveDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/oldWALs, maxLogs=10
2024-11-13T16:04:16,187 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40685%2C1731513856028.1731513856186
2024-11-13T16:04:16,192 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186
2024-11-13T16:04:16,196 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33141:33141),(127.0.0.1/127.0.0.1:46583:46583)]
2024-11-13T16:04:16,197 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:04:16,197 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:16,197 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,197 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,199 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:16,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:04:16,201 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,201 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:16,203 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:04:16,203 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,203 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:16,203 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:16,205 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:04:16,205 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,205 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:16,205 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:04:16,207 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:04:16,207 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,207 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:16,207 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,208 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,208 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,210 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,210 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,210 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:04:16,212 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:04:16,215 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:04:16,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=845259, jitterRate=0.0748031735420227}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:04:16,216 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513856198Initializing all the Stores at 1731513856199 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856199Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513856199Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513856199Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513856199Cleaning up temporary data from old regions at 1731513856210 (+11 ms)Region opened successfully at 1731513856216 (+6 ms)
2024-11-13T16:04:16,217 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:04:16,221 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54383506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:04:16,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:04:16,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:04:16,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:04:16,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:04:16,223 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:04:16,224 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:04:16,224 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:04:16,226 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:04:16,227 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:04:16,229 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:04:16,229 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:04:16,230 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:04:16,231 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:04:16,231 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:04:16,232 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:04:16,234 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:04:16,235 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:04:16,237 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:04:16,239 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:04:16,240 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:04:16,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:16,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:16,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,242 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,40685,1731513856028, sessionid=0x100a608402a0000, setting cluster-up flag (Was=false)
2024-11-13T16:04:16,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,253 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:04:16,254 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,264 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:04:16,265 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,40685,1731513856028
2024-11-13T16:04:16,266 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:04:16,268 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:16,268 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:04:16,268 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:04:16,269 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,40685,1731513856028
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:04:16,270 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,273 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731513886273
2024-11-13T16:04:16,273 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,274 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:16,274 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:04:16,274 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:04:16,275 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:04:16,275 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:04:16,276 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,276 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:04:16,278 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513856275,5,FailOnTimeoutGroup]
2024-11-13T16:04:16,278 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513856278,5,FailOnTimeoutGroup]
2024-11-13T16:04:16,278 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,278 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:04:16,278 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,279 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:04:16,286 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:04:16,287 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:04:16,287 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b
2024-11-13T16:04:16,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:04:16,295 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:04:16,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:16,297 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(746): ClusterId : a7ffe9c4-9f97-4bee-a1c9-e04ee1554e10
2024-11-13T16:04:16,297 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:04:16,298 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:04:16,300 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:04:16,300 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:04:16,300 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:04:16,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,301 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,301 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:04:16,302 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:04:16,303 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:04:16,303 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,303 DEBUG [RS:0;c4c650e80f58:41961 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf1fb71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:04:16,303 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,303 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:04:16,305 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:04:16,305 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,305 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,305 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:04:16,307 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:04:16,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,307 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:04:16,308 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740
2024-11-13T16:04:16,309 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740
2024-11-13T16:04:16,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:04:16,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:04:16,311 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:04:16,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:04:16,314 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:04:16,315 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698480, jitterRate=-0.11183780431747437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:04:16,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513856296Initializing all the Stores at 1731513856297 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856297Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856298 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513856298Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856298Cleaning up temporary data from old regions at 1731513856310 (+12 ms)Region opened successfully at 1731513856316 (+6 ms)
2024-11-13T16:04:16,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:04:16,316 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:04:16,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:04:16,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:04:16,316 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:04:16,355 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:16,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513856316Disabling compacts and flushes for region at 1731513856316Disabling writes for close at 1731513856316Writing region close event to WAL at 1731513856355 (+39 ms)Closed at 1731513856355
2024-11-13T16:04:16,357 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:41961
2024-11-13T16:04:16,357 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:04:16,357 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:04:16,357 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:04:16,358 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,40685,1731513856028 with port=41961, startcode=1731513856079
2024-11-13T16:04:16,358 DEBUG [RS:0;c4c650e80f58:41961 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:04:16,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:16,358 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:04:16,359 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:04:16,361 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:04:16,362 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46241, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:04:16,362 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,362 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40685 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,362 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:04:16,366 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b
2024-11-13T16:04:16,366 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46231
2024-11-13T16:04:16,366 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:04:16,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:16,370 DEBUG [RS:0;c4c650e80f58:41961 {}] zookeeper.ZKUtil(111): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,370 WARN  [RS:0;c4c650e80f58:41961 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:04:16,370 INFO  [RS:0;c4c650e80f58:41961 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:16,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:16,370 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:04:16,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling
2024-11-13T16:04:16,373 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,41961,1731513856079]
2024-11-13T16:04:16,376 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:04:16,379 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:04:16,383 INFO  [RS:0;c4c650e80f58:41961 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:04:16,383 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,384 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:04:16,386 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:04:16,386 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,386 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,387 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:16,387 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:16,387 DEBUG [RS:0;c4c650e80f58:41961 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,387 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41961,1731513856079-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:04:16,407 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:04:16,408 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41961,1731513856079-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,408 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,408 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.Replication(171): c4c650e80f58,41961,1731513856079 started
2024-11-13T16:04:16,422 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,422 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,41961,1731513856079, RpcServer on c4c650e80f58/172.17.0.3:41961, sessionid=0x100a608402a0001
2024-11-13T16:04:16,423 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:04:16,423 DEBUG [RS:0;c4c650e80f58:41961 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,423 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,41961,1731513856079'
2024-11-13T16:04:16,423 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:04:16,424 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:04:16,424 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:04:16,424 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:04:16,424 DEBUG [RS:0;c4c650e80f58:41961 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,425 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,41961,1731513856079'
2024-11-13T16:04:16,425 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:04:16,425 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:04:16,426 DEBUG [RS:0;c4c650e80f58:41961 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:04:16,426 INFO  [RS:0;c4c650e80f58:41961 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:04:16,426 INFO  [RS:0;c4c650e80f58:41961 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:04:16,513 WARN  [c4c650e80f58:40685 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:04:16,529 INFO  [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C41961%2C1731513856079, suffix=, logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079, archiveDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs, maxLogs=32
2024-11-13T16:04:16,530 INFO  [RS:0;c4c650e80f58:41961 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513856529
2024-11-13T16:04:16,538 INFO  [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529
2024-11-13T16:04:16,543 DEBUG [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46583:46583),(127.0.0.1/127.0.0.1:33141:33141)]
2024-11-13T16:04:16,591 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:16,597 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:16,763 DEBUG [c4c650e80f58:40685 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:04:16,764 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,765 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,41961,1731513856079, state=OPENING
2024-11-13T16:04:16,767 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:04:16,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:16,770 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:04:16,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:16,770 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:16,770 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,41961,1731513856079}]
2024-11-13T16:04:16,923 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:04:16,926 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:04:16,930 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:04:16,930 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:16,932 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C41961%2C1731513856079.meta, suffix=.meta, logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079, archiveDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs, maxLogs=32
2024-11-13T16:04:16,932 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
2024-11-13T16:04:16,938 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
2024-11-13T16:04:16,942 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46583:46583),(127.0.0.1/127.0.0.1:33141:33141)]
2024-11-13T16:04:16,943 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:04:16,944 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:04:16,944 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:04:16,946 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:04:16,947 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:04:16,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,948 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:04:16,949 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:04:16,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,950 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:04:16,951 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:04:16,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,951 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:04:16,952 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:04:16,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:16,953 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:04:16,953 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:04:16,954 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740
2024-11-13T16:04:16,955 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740
2024-11-13T16:04:16,956 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:04:16,956 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:04:16,957 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:04:16,958 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:04:16,959 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=729568, jitterRate=-0.07230664789676666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:04:16,959 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:04:16,959 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513856944Writing region info on filesystem at 1731513856944Initializing all the Stores at 1731513856945 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856945Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856946 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513856946Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513856946Cleaning up temporary data from old regions at 1731513856956 (+10 ms)Running coprocessor post-open hooks at 1731513856959 (+3 ms)Region opened successfully at 1731513856959
2024-11-13T16:04:16,961 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513856923
2024-11-13T16:04:16,964 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:04:16,964 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:04:16,965 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,966 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,41961,1731513856079, state=OPEN
2024-11-13T16:04:16,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:04:16,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:04:16,975 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:16,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:16,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:04:16,979 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:04:16,979 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,41961,1731513856079 in 205 msec
2024-11-13T16:04:16,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:04:16,982 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 620 msec
2024-11-13T16:04:16,983 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:04:16,983 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:04:16,984 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:04:16,984 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,41961,1731513856079, seqNum=-1]
2024-11-13T16:04:16,985 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:04:16,986 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:04:16,992 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 724 msec
2024-11-13T16:04:16,992 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513856992, completionTime=-1
2024-11-13T16:04:16,992 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:04:16,992 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:04:16,994 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:04:16,994 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731513916994
2024-11-13T16:04:16,994 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731513976994
2024-11-13T16:04:16,994 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:40685, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,995 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:16,997 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.881sec
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:04:16,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:04:17,002 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:04:17,002 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:04:17,002 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40685,1731513856028-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,098 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57ab428e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:17,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,40685,-1 for getting cluster id
2024-11-13T16:04:17,098 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:04:17,100 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7ffe9c4-9f97-4bee-a1c9-e04ee1554e10'
2024-11-13T16:04:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:04:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7ffe9c4-9f97-4bee-a1c9-e04ee1554e10"

2024-11-13T16:04:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed00ef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,40685,-1]
2024-11-13T16:04:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:04:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:17,103 INFO  [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:04:17,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af62680, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:04:17,104 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:04:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,41961,1731513856079, seqNum=-1]
2024-11-13T16:04:17,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:04:17,107 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:04:17,109 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,40685,1731513856028
2024-11-13T16:04:17,109 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:17,111 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:04:17,112 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:04:17,118 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:04:17,127 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:04:17,128 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34133
2024-11-13T16:04:17,129 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34133 connecting to ZooKeeper ensemble=127.0.0.1:49300
2024-11-13T16:04:17,130 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:17,132 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:04:17,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:341330x0, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:04:17,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/running
2024-11-13T16:04:17,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34133-0x100a608402a0002 connected
2024-11-13T16:04:17,137 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT
2024-11-13T16:04:17,138 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:04:17,140 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:17,143 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:04:17,143 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:17,144 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:04:17,144 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:17,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:04:17,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34133
2024-11-13T16:04:17,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34133
2024-11-13T16:04:17,151 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34133
2024-11-13T16:04:17,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34133
2024-11-13T16:04:17,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34133
2024-11-13T16:04:17,159 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(746): ClusterId : a7ffe9c4-9f97-4bee-a1c9-e04ee1554e10
2024-11-13T16:04:17,160 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:04:17,162 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:04:17,162 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:04:17,165 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:04:17,166 DEBUG [RS:1;c4c650e80f58:34133 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14e7ce92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:04:17,179 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c4c650e80f58:34133
2024-11-13T16:04:17,180 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:04:17,180 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:04:17,180 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:04:17,181 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,40685,1731513856028 with port=34133, startcode=1731513857127
2024-11-13T16:04:17,181 DEBUG [RS:1;c4c650e80f58:34133 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:04:17,184 INFO  [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45587, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:04:17,184 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,184 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40685 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,186 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b
2024-11-13T16:04:17,186 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46231
2024-11-13T16:04:17,186 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:04:17,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:17,188 DEBUG [RS:1;c4c650e80f58:34133 {}] zookeeper.ZKUtil(111): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,189 WARN  [RS:1;c4c650e80f58:34133 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:04:17,189 INFO  [RS:1;c4c650e80f58:34133 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:04:17,189 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,34133,1731513857127]
2024-11-13T16:04:17,189 DEBUG [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,192 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:04:17,198 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:04:17,198 INFO  [RS:1;c4c650e80f58:34133 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:04:17,198 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,199 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:04:17,200 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:04:17,200 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:17,200 DEBUG [RS:1;c4c650e80f58:34133 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,201 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,34133,1731513857127-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:04:17,217 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:04:17,217 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,34133,1731513857127-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,217 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,217 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.Replication(171): c4c650e80f58,34133,1731513857127 started
2024-11-13T16:04:17,231 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:04:17,232 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,34133,1731513857127, RpcServer on c4c650e80f58/172.17.0.3:34133, sessionid=0x100a608402a0002
2024-11-13T16:04:17,232 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:04:17,232 DEBUG [RS:1;c4c650e80f58:34133 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,232 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;c4c650e80f58:34133,5,FailOnTimeoutGroup]
2024-11-13T16:04:17,232 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,34133,1731513857127'
2024-11-13T16:04:17,232 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:04:17,232 INFO  [Time-limited test {}] wal.TestLogRolling(207): Replication=2
2024-11-13T16:04:17,233 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,34133,1731513857127
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,34133,1731513857127'
2024-11-13T16:04:17,233 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:04:17,234 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:04:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is c4c650e80f58,40685,1731513856028
2024-11-13T16:04:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4606f62b
2024-11-13T16:04:17,234 DEBUG [RS:1;c4c650e80f58:34133 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:04:17,234 INFO  [RS:1;c4c650e80f58:34133 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:04:17,234 INFO  [RS:1;c4c650e80f58:34133 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:04:17,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-13T16:04:17,236 INFO  [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-13T16:04:17,237 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-13T16:04:17,237 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-13T16:04:17,237 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:04:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath
2024-11-13T16:04:17,240 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION
2024-11-13T16:04:17,241 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:17,241 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnDatanodeDeath"
 procId is: 4
2024-11-13T16:04:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:04:17,242 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-13T16:04:17,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741835_1011 (size=393)
2024-11-13T16:04:17,250 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741835_1011 (size=393)
2024-11-13T16:04:17,253 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 536ac2cf66251fda0ce168375295a1d6, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b
2024-11-13T16:04:17,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46019 is added to blk_1073741836_1012 (size=76)
2024-11-13T16:04:17,260 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45619 is added to blk_1073741836_1012 (size=76)
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 536ac2cf66251fda0ce168375295a1d6, disabling compactions & flushes
2024-11-13T16:04:17,261 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6. after waiting 0 ms
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,261 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,261 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 536ac2cf66251fda0ce168375295a1d6:
Waiting for close lock at 1731513857261Disabling compacts and flushes for region at 1731513857261Disabling writes for close at 1731513857261Writing region close event to WAL at 1731513857261Closed at 1731513857261
2024-11-13T16:04:17,263 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META
2024-11-13T16:04:17,263 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731513857263"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731513857263"}]},"ts":"1731513857263"}
2024-11-13T16:04:17,266 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-13T16:04:17,267 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-13T16:04:17,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513857267"}]},"ts":"1731513857267"}
2024-11-13T16:04:17,270 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta
2024-11-13T16:04:17,270 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=536ac2cf66251fda0ce168375295a1d6, ASSIGN}]
2024-11-13T16:04:17,272 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=536ac2cf66251fda0ce168375295a1d6, ASSIGN
2024-11-13T16:04:17,273 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=536ac2cf66251fda0ce168375295a1d6, ASSIGN; state=OFFLINE, location=c4c650e80f58,41961,1731513856079; forceNewPlan=false, retain=false
2024-11-13T16:04:17,336 INFO  [RS:1;c4c650e80f58:34133 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C34133%2C1731513857127, suffix=, logDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127, archiveDir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs, maxLogs=32
2024-11-13T16:04:17,337 INFO  [RS:1;c4c650e80f58:34133 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C34133%2C1731513857127.1731513857337
2024-11-13T16:04:17,343 INFO  [RS:1;c4c650e80f58:34133 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
2024-11-13T16:04:17,344 DEBUG [RS:1;c4c650e80f58:34133 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46583:46583),(127.0.0.1/127.0.0.1:33141:33141)]
2024-11-13T16:04:17,424 INFO  [c4c650e80f58:40685 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 
2024-11-13T16:04:17,424 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=536ac2cf66251fda0ce168375295a1d6, regionState=OPENING, regionLocation=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:17,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=536ac2cf66251fda0ce168375295a1d6, ASSIGN because future has completed
2024-11-13T16:04:17,428 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 536ac2cf66251fda0ce168375295a1d6, server=c4c650e80f58,41961,1731513856079}]
2024-11-13T16:04:17,584 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,585 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 536ac2cf66251fda0ce168375295a1d6, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:04:17,585 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,585 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:04:17,585 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,585 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,587 INFO  [StoreOpener-536ac2cf66251fda0ce168375295a1d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 536ac2cf66251fda0ce168375295a1d6 
2024-11-13T16:04:17,588 INFO  [StoreOpener-536ac2cf66251fda0ce168375295a1d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 536ac2cf66251fda0ce168375295a1d6 columnFamilyName info
2024-11-13T16:04:17,589 DEBUG [StoreOpener-536ac2cf66251fda0ce168375295a1d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:04:17,589 INFO  [StoreOpener-536ac2cf66251fda0ce168375295a1d6-1 {}] regionserver.HStore(327): Store=536ac2cf66251fda0ce168375295a1d6/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:04:17,589 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,590 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,590 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,591 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,591 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,592 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,594 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:04:17,594 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 536ac2cf66251fda0ce168375295a1d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=862227, jitterRate=0.09637954831123352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:04:17,595 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:17,595 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 536ac2cf66251fda0ce168375295a1d6:
Running coprocessor pre-open hook at 1731513857586Writing region info on filesystem at 1731513857586Initializing all the Stores at 1731513857587 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513857587Cleaning up temporary data from old regions at 1731513857591 (+4 ms)Running coprocessor post-open hooks at 1731513857595 (+4 ms)Region opened successfully at 1731513857595
2024-11-13T16:04:17,596 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., pid=6, masterSystemTime=1731513857580
2024-11-13T16:04:17,599 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,599 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:17,600 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=536ac2cf66251fda0ce168375295a1d6, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,41961,1731513856079
2024-11-13T16:04:17,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 536ac2cf66251fda0ce168375295a1d6, server=c4c650e80f58,41961,1731513856079 because future has completed
2024-11-13T16:04:17,607 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-13T16:04:17,607 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 536ac2cf66251fda0ce168375295a1d6, server=c4c650e80f58,41961,1731513856079 in 176 msec
2024-11-13T16:04:17,609 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-13T16:04:17,609 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=536ac2cf66251fda0ce168375295a1d6, ASSIGN in 337 msec
2024-11-13T16:04:17,611 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-13T16:04:17,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513857611"}]},"ts":"1731513857611"}
2024-11-13T16:04:17,613 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta
2024-11-13T16:04:17,614 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION
2024-11-13T16:04:17,616 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 377 msec
2024-11-13T16:04:22,446 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:04:22,448 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:22,467 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:22,469 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:22,470 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:22,480 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath'
2024-11-13T16:04:26,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:04:26,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-13T16:04:26,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-13T16:04:26,371 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer
2024-11-13T16:04:26,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:26,372 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-13T16:04:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:04:27,344 INFO  [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed
2024-11-13T16:04:27,344 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100
2024-11-13T16:04:27,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath
2024-11-13T16:04:27,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:27,361 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:27,364 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:27,364 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:27,364 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:27,364 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:04:27,365 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@691515f5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:27,365 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2af8b745{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:27,480 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@25509568{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-32957-hadoop-hdfs-3_4_1-tests_jar-_-any-17202056925921088926/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:27,481 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5dc4ef73{HTTP/1.1, (http/1.1)}{localhost:32957}
2024-11-13T16:04:27,481 INFO  [Time-limited test {}] server.Server(415): Started @116367ms
2024-11-13T16:04:27,482 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:27,519 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:27,522 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:27,523 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:27,523 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:27,523 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:27,524 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3b9c8816{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:27,524 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@156f3a55{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:27,596 WARN  [Thread-828 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data5/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:27,596 WARN  [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data6/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:27,613 WARN  [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:27,615 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51650c508688cc9a with lease ID 0x7b55e8a9c370ad41: Processing first storage report for DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f from datanode DatanodeRegistration(127.0.0.1:44501, datanodeUuid=91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362, infoPort=33953, infoSecurePort=0, ipcPort=33831, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:27,615 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51650c508688cc9a with lease ID 0x7b55e8a9c370ad41: from storage DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f node DatanodeRegistration(127.0.0.1:44501, datanodeUuid=91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362, infoPort=33953, infoSecurePort=0, ipcPort=33831, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:04:27,616 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x51650c508688cc9a with lease ID 0x7b55e8a9c370ad41: Processing first storage report for DS-a6b25a20-271b-4db3-8b4e-ba7136772097 from datanode DatanodeRegistration(127.0.0.1:44501, datanodeUuid=91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362, infoPort=33953, infoSecurePort=0, ipcPort=33831, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:27,616 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x51650c508688cc9a with lease ID 0x7b55e8a9c370ad41: from storage DS-a6b25a20-271b-4db3-8b4e-ba7136772097 node DatanodeRegistration(127.0.0.1:44501, datanodeUuid=91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362, infoPort=33953, infoSecurePort=0, ipcPort=33831, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:27,651 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@412902c6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-46541-hadoop-hdfs-3_4_1-tests_jar-_-any-7258500877946130225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:27,652 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5bea65f7{HTTP/1.1, (http/1.1)}{localhost:46541}
2024-11-13T16:04:27,652 INFO  [Time-limited test {}] server.Server(415): Started @116538ms
2024-11-13T16:04:27,653 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:27,693 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:27,696 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:27,701 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:27,701 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:27,701 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:04:27,701 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e96eece{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:27,702 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@687696f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:27,864 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7d285dba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-35359-hadoop-hdfs-3_4_1-tests_jar-_-any-15359684377635075237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:27,865 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@158a9d8c{HTTP/1.1, (http/1.1)}{localhost:35359}
2024-11-13T16:04:27,865 INFO  [Time-limited test {}] server.Server(415): Started @116751ms
2024-11-13T16:04:27,866 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:28,092 WARN  [Thread-877 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data7/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:28,092 WARN  [Thread-878 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data8/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:28,114 WARN  [Thread-843 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:28,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa702b9b8a05d2135 with lease ID 0x7b55e8a9c370ad42: Processing first storage report for DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc from datanode DatanodeRegistration(127.0.0.1:41321, datanodeUuid=c2225620-e1f4-4f26-b9e1-ad86a505240c, infoPort=34819, infoSecurePort=0, ipcPort=36717, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:28,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa702b9b8a05d2135 with lease ID 0x7b55e8a9c370ad42: from storage DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc node DatanodeRegistration(127.0.0.1:41321, datanodeUuid=c2225620-e1f4-4f26-b9e1-ad86a505240c, infoPort=34819, infoSecurePort=0, ipcPort=36717, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:28,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa702b9b8a05d2135 with lease ID 0x7b55e8a9c370ad42: Processing first storage report for DS-7b245fb8-d93e-47aa-a47b-2c7e1fa3af88 from datanode DatanodeRegistration(127.0.0.1:41321, datanodeUuid=c2225620-e1f4-4f26-b9e1-ad86a505240c, infoPort=34819, infoSecurePort=0, ipcPort=36717, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:28,117 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa702b9b8a05d2135 with lease ID 0x7b55e8a9c370ad42: from storage DS-7b245fb8-d93e-47aa-a47b-2c7e1fa3af88 node DatanodeRegistration(127.0.0.1:41321, datanodeUuid=c2225620-e1f4-4f26-b9e1-ad86a505240c, infoPort=34819, infoSecurePort=0, ipcPort=36717, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:28,204 WARN  [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:28,204 WARN  [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10/current/BP-1920315224-172.17.0.3-1731513855204/current, will proceed with Du for space computation calculation, 
2024-11-13T16:04:28,221 WARN  [Thread-865 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:28,224 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a87c3d16baa401e with lease ID 0x7b55e8a9c370ad43: Processing first storage report for DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5 from datanode DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:28,224 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a87c3d16baa401e with lease ID 0x7b55e8a9c370ad43: from storage DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5 node DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:28,224 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a87c3d16baa401e with lease ID 0x7b55e8a9c370ad43: Processing first storage report for DS-fc070204-e1df-43db-a07a-d1166fdadc8e from datanode DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204)
2024-11-13T16:04:28,224 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a87c3d16baa401e with lease ID 0x7b55e8a9c370ad43: from storage DS-fc070204-e1df-43db-a07a-d1166fdadc8e node DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:28,295 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,295 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013
java.io.IOException: Bad response ERROR for BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,295 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta block BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:28,296 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:28,296 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009
java.io.IOException: Bad response ERROR for BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,297 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2062042356_22 at /127.0.0.1:49082 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:45619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49082 dst: /127.0.0.1:45619
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,297 WARN  [PacketResponder: BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46019] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:49014 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49014 dst: /127.0.0.1:45619
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:36012 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:46019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36012 dst: /127.0.0.1:46019
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,298 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b2e9f0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:28,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2062042356_22 at /127.0.0.1:36038 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:46019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36038 dst: /127.0.0.1:46019
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:35970 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:46019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:35970 dst: /127.0.0.1:46019
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,298 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2c64d82b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:28,298 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:28,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:49062 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49062 dst: /127.0.0.1:45619
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,298 WARN  [PacketResponder: BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:46019] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Broken pipe
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45dc61f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:28,299 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63c2c4fc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:28,299 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:36002 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:46019:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36002 dst: /127.0.0.1:46019
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,299 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:49052 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:45619:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49052 dst: /127.0.0.1:45619
java.net.SocketException: Connection reset
	at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?]
	at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:28,303 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:28,303 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:28,303 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid 02927f04-8187-4c41-87c8-0101e77f0f9c) service to localhost/127.0.0.1:46231
2024-11-13T16:04:28,303 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:28,303 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:28,303 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:28,303 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:28,304 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,310 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,311 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta block BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,311 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,315 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1827207e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:28,316 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a18c5e6{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:28,316 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:28,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2693ea96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:28,316 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e195dbd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:28,317 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:28,317 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid ec6664ce-658a-4dd3-b473-2ab8d4387437) service to localhost/127.0.0.1:46231
2024-11-13T16:04:28,317 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:28,317 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:28,318 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data1/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:28,318 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data2/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:28,318 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:28,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., hostname=c4c650e80f58,41961,1731513856079, seqNum=2]
2024-11-13T16:04:28,324 ERROR [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b-prefix:c4c650e80f58,41961,1731513856079 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,324 WARN  [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b-prefix:c4c650e80f58,41961,1731513856079 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,324 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,324 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C41961%2C1731513856079:(num 1731513856529) roll requested
2024-11-13T16:04:28,324 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513868324
2024-11-13T16:04:28,330 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:28,331 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:28,331 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:28,331 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:28,331 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:28,331 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324
2024-11-13T16:04:28,331 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,332 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:28,332 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34819:34819),(127.0.0.1/127.0.0.1:33953:33953)]
2024-11-13T16:04:28,332 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:28,333 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils
2024-11-13T16:04:28,333 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease()
2024-11-13T16:04:28,333 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529
2024-11-13T16:04:28,336 WARN  [IPC Server handler 2 on default port 46231 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009
2024-11-13T16:04:28,339 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 after 5ms
2024-11-13T16:04:29,201 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:30,163 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:30,333 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:30,334 INFO  [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324
2024-11-13T16:04:30,335 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:30,335 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:30,335 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:36048 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:41321:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36048 dst: /127.0.0.1:41321
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:30,336 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:54758 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:44501:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54758 dst: /127.0.0.1:44501
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:30,337 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@412902c6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:30,337 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5bea65f7{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:30,337 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:30,337 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@156f3a55{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:30,337 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3b9c8816{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:30,339 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:30,339 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:30,339 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid c2225620-e1f4-4f26-b9e1-ad86a505240c) service to localhost/127.0.0.1:46231
2024-11-13T16:04:30,339 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:30,339 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data7/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:30,340 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data8/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:30,340 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:31,202 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:32,163 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:32,333 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]
2024-11-13T16:04:32,333 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:32,334 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C41961%2C1731513856079:(num 1731513868324) roll requested
2024-11-13T16:04:32,334 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513872334
2024-11-13T16:04:32,337 WARN  [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:32,338 WARN  [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:32,338 WARN  [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741839_1021
2024-11-13T16:04:32,340 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 after 4007ms
2024-11-13T16:04:32,340 WARN  [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:32,345 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1])
2024-11-13T16:04:32,347 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:32,347 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:32,347 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:32,347 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:32,348 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:32,348 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513872334
2024-11-13T16:04:32,349 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33953:33953),(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:32,349 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:32,349 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324 is not closed yet, will try archiving it next time
2024-11-13T16:04:32,350 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44501 is added to blk_1073741838_1020 (size=3600)
2024-11-13T16:04:32,751 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:33,202 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:33,628 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@1dd71ed8[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44501, datanodeUuid=91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362, infoPort=33953, infoSecurePort=0, ipcPort=33831, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741838_1020 to 127.0.0.1:41321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:34,164 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,349 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,350 WARN  [ResponseProcessor for block BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,350 WARN  [DataStreamer for file /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513872334 block BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:34,351 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:36462 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:44501:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:36462 dst: /127.0.0.1:44501
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:34,351 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41828 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41828 dst: /127.0.0.1:33387
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:34,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@25509568{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:34,353 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5dc4ef73{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:04:34,353 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:04:34,353 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2af8b745{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:04:34,354 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@691515f5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:04:34,355 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:04:34,355 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:04:34,355 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid 91c23b5f-5fbe-4e9f-8f2e-9dbc5b0b4362) service to localhost/127.0.0.1:46231
2024-11-13T16:04:34,355 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:04:34,356 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data5/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:34,356 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data6/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:04:34,356 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:04:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:34,365 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:04:34,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4602593b92ba40f7b86cdcd6fbbde6db is 1080, key is row0002/info:/1731513870341/Put/seqid=0
2024-11-13T16:04:34,390 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41848 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:34,390 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:34,390 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024
2024-11-13T16:04:34,390 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41848 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:34,390 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41848 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741841_1024] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41848 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:34,391 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:34,392 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,392 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:34,392 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741842_1025
2024-11-13T16:04:34,393 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:34,394 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,394 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:34,394 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741843_1026
2024-11-13T16:04:34,395 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:34,396 WARN  [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:34,397 WARN  [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:34,397 WARN  [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741844_1027
2024-11-13T16:04:34,397 WARN  [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:34,398 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:34,398 WARN  [IPC Server handler 2 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:34,398 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:34,405 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741845_1028 (size=10347)
2024-11-13T16:04:34,807 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4602593b92ba40f7b86cdcd6fbbde6db
2024-11-13T16:04:34,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4602593b92ba40f7b86cdcd6fbbde6db as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db
2024-11-13T16:04:34,821 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db, entries=5, sequenceid=11, filesize=10.1 K
2024-11-13T16:04:34,822 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 536ac2cf66251fda0ce168375295a1d6 in 457ms, sequenceid=11, compaction requested=false
2024-11-13T16:04:34,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:34,988 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB
2024-11-13T16:04:34,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/eafec76df82440238b188c2ef28c479f is 1080, key is row0007/info:/1731513874366/Put/seqid=0
2024-11-13T16:04:35,000 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:35,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41876 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:35,000 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:35,000 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029
2024-11-13T16:04:35,000 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41876 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:35,000 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41876 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41876 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:35,001 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:35,003 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:35,003 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:35,003 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741847_1030
2024-11-13T16:04:35,004 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:35,011 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:35,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41888 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:35,011 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:35,012 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031
2024-11-13T16:04:35,012 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41888 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:35,012 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41888 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41888 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:35,012 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:35,013 WARN  [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:35,014 WARN  [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:35,014 WARN  [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741849_1032
2024-11-13T16:04:35,014 WARN  [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:35,015 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:35,015 WARN  [IPC Server handler 2 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:35,015 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:35,018 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741850_1033 (size=12506)
2024-11-13T16:04:35,203 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:35,419 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/eafec76df82440238b188c2ef28c479f
2024-11-13T16:04:35,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/eafec76df82440238b188c2ef28c479f as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f
2024-11-13T16:04:35,436 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f, entries=7, sequenceid=24, filesize=12.2 K
2024-11-13T16:04:35,438 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 536ac2cf66251fda0ce168375295a1d6 in 450ms, sequenceid=24, compaction requested=false
2024-11-13T16:04:35,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:35,438 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:35,438 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:35,439 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f because midkey is the same as first or last row
2024-11-13T16:04:36,164 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,349 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]
2024-11-13T16:04:36,350 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,350 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C41961%2C1731513856079:(num 1731513872334) roll requested
2024-11-13T16:04:36,350 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513876350
2024-11-13T16:04:36,354 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,354 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:36,354 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741851_1034
2024-11-13T16:04:36,355 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:36,359 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45619
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,359 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41900 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035 to mirror 127.0.0.1:45619
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,360 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:36,360 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035
2024-11-13T16:04:36,360 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41900 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-13T16:04:36,360 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41900 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41900 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,361 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:36,362 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,362 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:36,362 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741853_1036
2024-11-13T16:04:36,363 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:36,365 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41906 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,366 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41906 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-13T16:04:36,366 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41906 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41906 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,366 WARN  [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,367 WARN  [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:36,367 WARN  [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741854_1037
2024-11-13T16:04:36,367 WARN  [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:36,371 WARN  [IPC Server handler 1 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:36,371 WARN  [IPC Server handler 1 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:36,371 WARN  [IPC Server handler 1 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:36,376 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:36,376 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:36,376 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:36,377 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:36,377 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:36,377 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513872334 with entries=24, filesize=24.23 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513876350
2024-11-13T16:04:36,379 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741840_1023 (size=24823)
2024-11-13T16:04:36,386 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:36,386 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:36,386 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513872334 is not closed yet, will try archiving it next time
2024-11-13T16:04:36,389 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513868324 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C41961%2C1731513856079.1731513868324
2024-11-13T16:04:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:36,411 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-13T16:04:36,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/063a75c3472a4d15adee890cd42ed793 is 1079, key is tmprow/info:/1731513876410/Put/seqid=0
2024-11-13T16:04:36,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41934 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,422 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,422 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41934 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:36,422 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:36,422 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039
2024-11-13T16:04:36,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41934 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41934 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,423 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:36,424 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,424 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:36,424 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741857_1040
2024-11-13T16:04:36,425 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:36,426 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,426 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:36,426 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741858_1041
2024-11-13T16:04:36,427 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:36,429 WARN  [Thread-937 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41938 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,429 WARN  [Thread-937 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:36,429 WARN  [Thread-937 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042
2024-11-13T16:04:36,429 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41938 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042] {}] datanode.BlockReceiver(316): Block 1073741859 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:36,429 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41938 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741859_1042] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41938 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,429 WARN  [Thread-937 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:36,430 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:36,430 WARN  [IPC Server handler 4 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:36,430 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:36,433 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741860_1043 (size=6027)
2024-11-13T16:04:36,434 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/063a75c3472a4d15adee890cd42ed793
2024-11-13T16:04:36,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/063a75c3472a4d15adee890cd42ed793 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793
2024-11-13T16:04:36,447 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793, entries=1, sequenceid=34, filesize=5.9 K
2024-11-13T16:04:36,449 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 536ac2cf66251fda0ce168375295a1d6 in 37ms, sequenceid=34, compaction requested=true
2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K
2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f because midkey is the same as first or last row
2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 536ac2cf66251fda0ce168375295a1d6:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:04:36,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:36,450 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:04:36,451 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:04:36,451 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1541): 536ac2cf66251fda0ce168375295a1d6/info is initiating minor compaction (all files)
2024-11-13T16:04:36,451 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 536ac2cf66251fda0ce168375295a1d6/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:36,451 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793] into tmpdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp, totalSize=28.2 K
2024-11-13T16:04:36,452 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4602593b92ba40f7b86cdcd6fbbde6db, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731513870341
2024-11-13T16:04:36,452 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting eafec76df82440238b188c2ef28c479f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731513874366
2024-11-13T16:04:36,453 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting 063a75c3472a4d15adee890cd42ed793, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731513876410
2024-11-13T16:04:36,468 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 536ac2cf66251fda0ce168375295a1d6#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:04:36,468 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/7ce64e3d86ba4857a1f47b8541070328 is 1080, key is row0002/info:/1731513870341/Put/seqid=0
2024-11-13T16:04:36,470 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,471 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:36,471 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741861_1044
2024-11-13T16:04:36,471 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:36,474 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45619
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,474 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41972 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045 to mirror 127.0.0.1:45619
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,474 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:36,474 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045
2024-11-13T16:04:36,474 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41972 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:36,474 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41972 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41972 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,475 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:36,477 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41982 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,477 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:36,477 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046
2024-11-13T16:04:36,477 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41982 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:36,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41982 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41982 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,478 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:36,483 WARN  [Thread-945 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:36,483 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41984 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,483 WARN  [Thread-945 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:36,483 WARN  [Thread-945 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047
2024-11-13T16:04:36,484 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41984 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:36,484 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:41984 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41984 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:36,484 WARN  [Thread-945 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:36,485 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:36,485 WARN  [IPC Server handler 4 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:36,485 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:36,488 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741865_1048 (size=17994)
2024-11-13T16:04:36,780 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:36,780 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513872334 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C41961%2C1731513856079.1731513872334
2024-11-13T16:04:36,896 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/7ce64e3d86ba4857a1f47b8541070328 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328
2024-11-13T16:04:36,904 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 536ac2cf66251fda0ce168375295a1d6/info of 536ac2cf66251fda0ce168375295a1d6 into 7ce64e3d86ba4857a1f47b8541070328(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:04:36,904 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:36,904 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., storeName=536ac2cf66251fda0ce168375295a1d6/info, priority=13, startTime=1731513876449; duration=0sec
2024-11-13T16:04:36,904 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-13T16:04:36,904 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:36,904 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 because midkey is the same as first or last row
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 because midkey is the same as first or last row
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 because midkey is the same as first or last row
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:36,905 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 536ac2cf66251fda0ce168375295a1d6:info
2024-11-13T16:04:37,203 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:37,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c4b3609[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741845_1028 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:37,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@428cd1f0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741850_1033 to 127.0.0.1:41321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:37,835 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-13T16:04:37,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/b84ff9822df84143b12d75d56ec41415 is 1079, key is tmprow/info:/1731513877833/Put/seqid=0
2024-11-13T16:04:37,842 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:37,842 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:37,842 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741866_1049
2024-11-13T16:04:37,843 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:37,844 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:37,844 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:37,844 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741867_1050
2024-11-13T16:04:37,845 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:37,846 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:37,846 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:37,846 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741868_1051
2024-11-13T16:04:37,847 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:37,848 WARN  [Thread-956 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:37,848 WARN  [Thread-956 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:37,848 WARN  [Thread-956 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741869_1052
2024-11-13T16:04:37,848 WARN  [Thread-956 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:37,849 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:37,849 WARN  [IPC Server handler 4 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:37,849 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:37,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741870_1053 (size=6027)
2024-11-13T16:04:38,164 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c4b3609[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741860_1043 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:38,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@428cd1f0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741840_1023 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:38,261 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/b84ff9822df84143b12d75d56ec41415
2024-11-13T16:04:38,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/b84ff9822df84143b12d75d56ec41415 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415
2024-11-13T16:04:38,276 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415, entries=1, sequenceid=45, filesize=5.9 K
2024-11-13T16:04:38,277 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 536ac2cf66251fda0ce168375295a1d6 in 442ms, sequenceid=45, compaction requested=false
2024-11-13T16:04:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K
2024-11-13T16:04:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:38,277 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 because midkey is the same as first or last row
2024-11-13T16:04:38,389 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas.  Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]
2024-11-13T16:04:38,390 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,390 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C41961%2C1731513856079:(num 1731513876350) roll requested
2024-11-13T16:04:38,390 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513878390
2024-11-13T16:04:38,393 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,393 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:38,394 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741871_1054
2024-11-13T16:04:38,394 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:38,395 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,396 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:38,396 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741872_1055
2024-11-13T16:04:38,396 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:38,397 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,397 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:38,397 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741873_1056
2024-11-13T16:04:38,400 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:38,403 WARN  [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:38,403 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:42016 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:38,403 WARN  [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:38,403 WARN  [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057
2024-11-13T16:04:38,403 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:42016 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057] {}] datanode.BlockReceiver(316): Block 1073741874 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-13T16:04:38,403 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:42016 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741874_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:42016 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:38,404 WARN  [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:38,405 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:38,405 WARN  [IPC Server handler 4 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:38,405 WARN  [IPC Server handler 4 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:38,412 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:38,413 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:38,413 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:38,413 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:38,413 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:38,413 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513876350 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513878390
2024-11-13T16:04:38,415 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741855_1038 (size=13591)
2024-11-13T16:04:38,420 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:38,420 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:38,420 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513876350 is not closed yet, will try archiving it next time
2024-11-13T16:04:38,816 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 is not closed yet, will try archiving it next time
2024-11-13T16:04:39,203 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:39,259 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-13T16:04:39,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f1b8866606d943e5bc2b73954c357252 is 1079, key is tmprow/info:/1731513879257/Put/seqid=0
2024-11-13T16:04:39,267 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,267 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:39,267 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741876_1059
2024-11-13T16:04:39,268 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:39,270 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,270 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39560 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,270 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:39,270 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060
2024-11-13T16:04:39,270 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39560 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:39,270 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39560 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39560 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,271 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:39,273 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,273 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39568 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,273 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:39,273 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061
2024-11-13T16:04:39,273 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39568 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061] {}] datanode.BlockReceiver(316): Block 1073741878 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:39,273 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39568 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741878_1061] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39568 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,274 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:39,275 WARN  [Thread-965 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,275 WARN  [Thread-965 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:39,275 WARN  [Thread-965 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741879_1062
2024-11-13T16:04:39,275 WARN  [Thread-965 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:39,276 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:39,276 WARN  [IPC Server handler 2 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:39,276 WARN  [IPC Server handler 2 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:39,280 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741880_1063 (size=6027)
2024-11-13T16:04:39,681 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f1b8866606d943e5bc2b73954c357252
2024-11-13T16:04:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f1b8866606d943e5bc2b73954c357252 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252
2024-11-13T16:04:39,693 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252, entries=1, sequenceid=55, filesize=5.9 K
2024-11-13T16:04:39,694 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 536ac2cf66251fda0ce168375295a1d6 in 435ms, sequenceid=55, compaction requested=true
2024-11-13T16:04:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 because midkey is the same as first or last row
2024-11-13T16:04:39,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 536ac2cf66251fda0ce168375295a1d6:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:04:39,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:39,695 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:04:39,696 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:04:39,696 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1541): 536ac2cf66251fda0ce168375295a1d6/info is initiating minor compaction (all files)
2024-11-13T16:04:39,696 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 536ac2cf66251fda0ce168375295a1d6/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:39,696 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252] into tmpdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp, totalSize=29.3 K
2024-11-13T16:04:39,696 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ce64e3d86ba4857a1f47b8541070328, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731513870341
2024-11-13T16:04:39,697 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting b84ff9822df84143b12d75d56ec41415, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731513877833
2024-11-13T16:04:39,697 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b8866606d943e5bc2b73954c357252, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731513879257
2024-11-13T16:04:39,711 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 536ac2cf66251fda0ce168375295a1d6#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:04:39,712 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/c681067c3e34454793b6c6fd43de14c4 is 1080, key is row0002/info:/1731513870341/Put/seqid=0
2024-11-13T16:04:39,713 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,713 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:39,713 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741881_1064
2024-11-13T16:04:39,714 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:39,715 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,715 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:39,715 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741882_1065
2024-11-13T16:04:39,715 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:39,716 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,717 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:39,717 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741883_1066
2024-11-13T16:04:39,717 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:39,719 WARN  [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46019
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:39,719 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39588 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067 to mirror 127.0.0.1:46019
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,719 WARN  [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]) is bad.
2024-11-13T16:04:39,719 WARN  [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067
2024-11-13T16:04:39,719 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39588 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067] {}] datanode.BlockReceiver(316): Block 1073741884 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:39,719 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:39588 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741884_1067] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39588 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:39,720 WARN  [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46019,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]
2024-11-13T16:04:39,720 WARN  [IPC Server handler 1 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology
2024-11-13T16:04:39,720 WARN  [IPC Server handler 1 on default port 46231 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]})
2024-11-13T16:04:39,721 WARN  [IPC Server handler 1 on default port 46231 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable:  unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}
2024-11-13T16:04:39,723 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741885_1068 (size=18097)
2024-11-13T16:04:40,132 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/c681067c3e34454793b6c6fd43de14c4 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4
2024-11-13T16:04:40,139 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 536ac2cf66251fda0ce168375295a1d6/info of 536ac2cf66251fda0ce168375295a1d6 into c681067c3e34454793b6c6fd43de14c4(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:40,140 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., storeName=536ac2cf66251fda0ce168375295a1d6/info, priority=13, startTime=1731513879694; duration=0sec
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 because midkey is the same as first or last row
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 because midkey is the same as first or last row
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 because midkey is the same as first or last row
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:40,140 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 536ac2cf66251fda0ce168375295a1d6:info
2024-11-13T16:04:40,165 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:40,226 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c4b3609[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741870_1053 to 127.0.0.1:44501 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:40,226 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@428cd1f0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741865_1048 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:40,420 WARN  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas.
2024-11-13T16:04:40,420 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:40,485 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:04:40,489 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:04:40,490 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:04:40,490 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:04:40,490 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:04:40,491 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@236c96ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:04:40,491 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@61815e22{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:04:40,607 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@117da513{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/java.io.tmpdir/jetty-localhost-33689-hadoop-hdfs-3_4_1-tests_jar-_-any-15105847344425407470/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:04:40,608 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6842affb{HTTP/1.1, (http/1.1)}{localhost:33689}
2024-11-13T16:04:40,608 INFO  [Time-limited test {}] server.Server(415): Started @129493ms
2024-11-13T16:04:40,609 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:04:40,720 WARN  [Thread-990 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:04:40,731 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d30d8bdedffe38c with lease ID 0x7b55e8a9c370ad44: from storage DS-31719626-8e9c-4b40-b0e8-d9156c01922a node DatanodeRegistration(127.0.0.1:41913, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=36275, infoSecurePort=0, ipcPort=39269, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:40,732 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d30d8bdedffe38c with lease ID 0x7b55e8a9c370ad44: from storage DS-29d37a25-747e-405c-bdeb-2f855b4990ff node DatanodeRegistration(127.0.0.1:41913, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=36275, infoSecurePort=0, ipcPort=39269, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:04:41,204 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:41,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@428cd1f0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741855_1038 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:41,229 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741880_1063 (size=6027)
2024-11-13T16:04:42,165 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:42,420 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:43,204 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:43,225 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c4b3609[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741885_1068 to 127.0.0.1:41321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:44,166 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:44,421 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:45,205 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,008 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:04:46,166 INFO  [master:store-WAL-Roller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,275 ERROR [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData-prefix:c4c650e80f58,40685,1731513856028 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,275 WARN  [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData-prefix:c4c650e80f58,40685,1731513856028 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,275 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C40685%2C1731513856028:(num 1731513856186) roll requested
2024-11-13T16:04:46,275 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40685%2C1731513856028.1731513886275
2024-11-13T16:04:46,279 WARN  [Thread-1011 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,279 WARN  [Thread-1011 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:46,279 WARN  [Thread-1011 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741886_1069
2024-11-13T16:04:46,280 WARN  [Thread-1011 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:46,281 WARN  [Thread-1011 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741887_1070
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,281 WARN  [Thread-1011 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741887_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:46,281 WARN  [Thread-1011 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741887_1070
2024-11-13T16:04:46,281 WARN  [Thread-1011 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:46,283 WARN  [Thread-1011 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1071
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,283 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:39598 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:46,283 WARN  [Thread-1011 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:46,284 WARN  [Thread-1011 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071
2024-11-13T16:04:46,284 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:39598 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-13T16:04:46,284 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:39598 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741888_1071] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:39598 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:46,284 WARN  [Thread-1011 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:46,289 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:46,289 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:46,289 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:46,289 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:46,289 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:46,290 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513886275
2024-11-13T16:04:46,290 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,290 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:46,290 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186
2024-11-13T16:04:46,291 WARN  [IPC Server handler 2 on default port 46231 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741830_1006
2024-11-13T16:04:46,291 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 after 0ms
2024-11-13T16:04:46,292 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36275:36275),(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:46,292 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 is not closed yet, will try archiving it next time
2024-11-13T16:04:46,421 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:47,205 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:48,422 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:49,206 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:50,293 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 after 4002ms
2024-11-13T16:04:50,422 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:50,745 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@26ef246d {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:45619,null,null])
java.net.ConnectException: Call From c4c650e80f58/172.17.0.3 to localhost:37367 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-13T16:04:50,747 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741833_1019 (size=455)
2024-11-13T16:04:51,206 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:51,359 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513856529 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C41961%2C1731513856079.1731513856529
2024-11-13T16:04:51,360 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513876350 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C41961%2C1731513856079.1731513876350
2024-11-13T16:04:51,728 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@170302bb[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:41913, datanodeUuid=02927f04-8187-4c41-87c8-0101e77f0f9c, infoPort=36275, infoSecurePort=0, ipcPort=39269, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741833_1019 to 127.0.0.1:41321 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:52,422 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,206 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,642 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.1731513893642
2024-11-13T16:04:53,645 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,645 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:53,645 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741890_1074
2024-11-13T16:04:53,646 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:53,648 WARN  [Thread-1022 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741891_1075
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,648 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:43546 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4]'}, localName='127.0.0.1:41913', datanodeUuid='02927f04-8187-4c41-87c8-0101e77f0f9c', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,649 WARN  [Thread-1022 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:53,649 WARN  [Thread-1022 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075
2024-11-13T16:04:53,649 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:43546 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075] {}] datanode.BlockReceiver(316): Block 1073741891 has not released the reserved bytes. Releasing 268435456 bytes as part of close.
2024-11-13T16:04:53,649 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2061565597_22 at /127.0.0.1:43546 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741891_1075] {}] datanode.DataXceiver(331): 127.0.0.1:41913:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43546 dst: /127.0.0.1:41913
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,649 WARN  [Thread-1022 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:53,653 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:53,653 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:53,653 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:53,654 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:53,654 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:53,654 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513878390 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513893642
2024-11-13T16:04:53,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36275:36275),(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:53,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513878390 is not closed yet, will try archiving it next time
2024-11-13T16:04:53,655 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741875_1058 (size=12911)
2024-11-13T16:04:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:53,660 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB
2024-11-13T16:04:53,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/c42b05c24fc64144af3e313ded61df50 is 1080, key is row0013/info:/1731513893656/Put/seqid=0
2024-11-13T16:04:53,668 WARN  [Thread-1029 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56196 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,668 WARN  [Thread-1029 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:53,668 WARN  [Thread-1029 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077
2024-11-13T16:04:53,668 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56196 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:53,668 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56196 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56196 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,668 WARN  [Thread-1029 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:53,671 WARN  [Thread-1029 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741894_1078
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,670 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43562 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4]'}, localName='127.0.0.1:41913', datanodeUuid='02927f04-8187-4c41-87c8-0101e77f0f9c', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,671 WARN  [Thread-1029 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:53,671 WARN  [Thread-1029 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078
2024-11-13T16:04:53,671 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43562 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078] {}] datanode.BlockReceiver(316): Block 1073741894 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:53,671 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43562 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741894_1078] {}] datanode.DataXceiver(331): 127.0.0.1:41913:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43562 dst: /127.0.0.1:41913
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,671 WARN  [Thread-1029 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:53,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741895_1079 (size=8190)
2024-11-13T16:04:53,680 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741895_1079 (size=8190)
2024-11-13T16:04:53,681 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/c42b05c24fc64144af3e313ded61df50
2024-11-13T16:04:53,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/c42b05c24fc64144af3e313ded61df50 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50
2024-11-13T16:04:53,694 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50, entries=3, sequenceid=66, filesize=8.0 K
2024-11-13T16:04:53,695 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 536ac2cf66251fda0ce168375295a1d6 in 35ms, sequenceid=66, compaction requested=false
2024-11-13T16:04:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K
2024-11-13T16:04:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 because midkey is the same as first or last row
2024-11-13T16:04:53,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41961 {}] regionserver.HRegion(8855): Flush requested on 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:53,883 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-13T16:04:53,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4e1793c571484b688c87741d8f5009e7 is 1080, key is row0015/info:/1731513893661/Put/seqid=0
2024-11-13T16:04:53,889 WARN  [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1080
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,890 WARN  [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741896_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:53,890 WARN  [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741896_1080
2024-11-13T16:04:53,890 WARN  [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:53,893 WARN  [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,893 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56226 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,893 WARN  [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:53,893 WARN  [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081
2024-11-13T16:04:53,893 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56226 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:53,893 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56226 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741897_1081] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56226 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,893 WARN  [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:53,894 WARN  [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1082
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,894 WARN  [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741898_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:53,894 WARN  [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741898_1082
2024-11-13T16:04:53,895 WARN  [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:53,899 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741899_1083 (size=14660)
2024-11-13T16:04:53,900 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741899_1083 (size=14660)
2024-11-13T16:04:53,900 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4e1793c571484b688c87741d8f5009e7
2024-11-13T16:04:53,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/4e1793c571484b688c87741d8f5009e7 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7
2024-11-13T16:04:53,913 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7, entries=9, sequenceid=79, filesize=14.3 K
2024-11-13T16:04:53,914 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 536ac2cf66251fda0ce168375295a1d6 in 31ms, sequenceid=79, compaction requested=true
2024-11-13T16:04:53,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K
2024-11-13T16:04:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 because midkey is the same as first or last row
2024-11-13T16:04:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 536ac2cf66251fda0ce168375295a1d6:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:04:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:53,915 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:04:53,916 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:04:53,916 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1541): 536ac2cf66251fda0ce168375295a1d6/info is initiating minor compaction (all files)
2024-11-13T16:04:53,916 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 536ac2cf66251fda0ce168375295a1d6/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:53,916 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7] into tmpdir=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp, totalSize=40.0 K
2024-11-13T16:04:53,917 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting c681067c3e34454793b6c6fd43de14c4, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731513870341
2024-11-13T16:04:53,917 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42b05c24fc64144af3e313ded61df50, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731513880273
2024-11-13T16:04:53,917 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e1793c571484b688c87741d8f5009e7, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731513893661
2024-11-13T16:04:53,930 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 536ac2cf66251fda0ce168375295a1d6#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:04:53,931 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/9019be82a4d84ac58fad635e062083a4 is 1080, key is row0002/info:/1731513870341/Put/seqid=0
2024-11-13T16:04:53,933 WARN  [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1084
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,934 WARN  [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741900_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:53,934 WARN  [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741900_1084
2024-11-13T16:04:53,934 WARN  [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:53,936 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56254 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,936 WARN  [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741901_1085
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,937 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56254 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085] {}] datanode.BlockReceiver(316): Block 1073741901 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:53,937 WARN  [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:53,937 WARN  [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085
2024-11-13T16:04:53,937 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56254 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741901_1085] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56254 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,937 WARN  [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:53,940 WARN  [Thread-1047 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1086
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:53,940 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56260 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,940 WARN  [Thread-1047 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:53,940 WARN  [Thread-1047 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086
2024-11-13T16:04:53,940 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56260 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086] {}] datanode.BlockReceiver(316): Block 1073741902 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:53,940 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56260 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741902_1086] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56260 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:53,940 WARN  [Thread-1047 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:53,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741903_1087 (size=28989)
2024-11-13T16:04:53,945 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741903_1087 (size=28989)
2024-11-13T16:04:53,952 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/9019be82a4d84ac58fad635e062083a4 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/9019be82a4d84ac58fad635e062083a4
2024-11-13T16:04:53,960 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 536ac2cf66251fda0ce168375295a1d6/info of 536ac2cf66251fda0ce168375295a1d6 into 9019be82a4d84ac58fad635e062083a4(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:04:53,960 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 536ac2cf66251fda0ce168375295a1d6:

2024-11-13T16:04:53,961 INFO  [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., storeName=536ac2cf66251fda0ce168375295a1d6/info, priority=13, startTime=1731513893915; duration=0sec
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/9019be82a4d84ac58fad635e062083a4 because midkey is the same as first or last row
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/9019be82a4d84ac58fad635e062083a4 because midkey is the same as first or last row
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/9019be82a4d84ac58fad635e062083a4 because midkey is the same as first or last row
2024-11-13T16:04:53,961 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:04:53,962 DEBUG [RS:0;c4c650e80f58:41961-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 536ac2cf66251fda0ce168375295a1d6:info
2024-11-13T16:04:54,056 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.1731513878390 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C41961%2C1731513856079.1731513878390
2024-11-13T16:04:54,423 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled.
2024-11-13T16:04:54,423 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.FSHLog(580): 
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,490 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:04:54,490 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:54,490 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:54,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:54,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:54,491 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:04:54,491 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:04:54,491 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1650848494, stopped=false
2024-11-13T16:04:54,491 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,40685,1731513856028
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:54,493 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:54,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:54,494 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:04:54,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:54,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:54,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:54,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:54,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:04:54,494 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,41961,1731513856079' *****
2024-11-13T16:04:54,494 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:04:54,494 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,34133,1731513857127' *****
2024-11-13T16:04:54,494 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:04:54,495 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:04:54,495 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,34133,1731513857127
2024-11-13T16:04:54,495 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:04:54,495 INFO  [RS:0;c4c650e80f58:41961 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:54,495 INFO  [RS:0;c4c650e80f58:41961 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:04:54,495 INFO  [RS:1;c4c650e80f58:34133 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c4c650e80f58:34133.
2024-11-13T16:04:54,495 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(3091): Received CLOSE for 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:54,495 DEBUG [RS:1;c4c650e80f58:34133 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:54,495 DEBUG [RS:1;c4c650e80f58:34133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:54,496 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,34133,1731513857127; all regions closed.
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,41961,1731513856079
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:41961.
2024-11-13T16:04:54,496 DEBUG [RS:0;c4c650e80f58:41961 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:04:54,496 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 536ac2cf66251fda0ce168375295a1d6, disabling compactions & flushes
2024-11-13T16:04:54,496 DEBUG [RS:0;c4c650e80f58:41961 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:54,496 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:54,496 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:04:54,496 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6. after waiting 0 ms
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:04:54,496 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:04:54,496 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 536ac2cf66251fda0ce168375295a1d6 1/1 column families, dataSize=2.10 KB heapSize=2.50 KB
2024-11-13T16:04:54,496 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,496 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-13T16:04:54,496 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,496 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 536ac2cf66251fda0ce168375295a1d6=TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.}
2024-11-13T16:04:54,496 DEBUG [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 536ac2cf66251fda0ce168375295a1d6
2024-11-13T16:04:54,497 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,497 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,497 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:04:54,497 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,497 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:04:54,497 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:04:54,497 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:04:54,497 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:04:54,497 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB
2024-11-13T16:04:54,497 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,498 ERROR [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b-prefix:c4c650e80f58,41961,1731513856079.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,498 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,498 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
2024-11-13T16:04:54,498 WARN  [FSHLog-0-hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b-prefix:c4c650e80f58,41961,1731513856079.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,498 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C41961%2C1731513856079.meta:.meta(num 1731513856932) roll requested
2024-11-13T16:04:54,498 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41961%2C1731513856079.meta.1731513894498.meta
2024-11-13T16:04:54,498 WARN  [IPC Server handler 3 on default port 46231 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 has not been closed. Lease recovery is in progress. RecoveryId = 1088 for block blk_1073741837_1013
2024-11-13T16:04:54,499 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 after 1ms
2024-11-13T16:04:54,501 WARN  [Thread-1055 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1089
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,501 WARN  [Thread-1055 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741904_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:54,501 WARN  [Thread-1055 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741904_1089
2024-11-13T16:04:54,502 WARN  [Thread-1055 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:54,502 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f3e543082c4d4627853d9eae80567129 is 1079, key is tmprow/info:/1731513894288/Put/seqid=0
2024-11-13T16:04:54,504 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741906_1091
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,504 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741906_1091 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:54,504 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741906_1091
2024-11-13T16:04:54,505 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:54,507 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1092
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,508 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:54,507 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43608 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4]'}, localName='127.0.0.1:41913', datanodeUuid='02927f04-8187-4c41-87c8-0101e77f0f9c', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,508 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092
2024-11-13T16:04:54,508 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43608 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092] {}] datanode.BlockReceiver(316): Block 1073741907 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:54,508 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43608 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741907_1092] {}] datanode.DataXceiver(331): 127.0.0.1:41913:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43608 dst: /127.0.0.1:41913
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,508 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:54,510 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,510 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,510 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,510 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,510 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,511 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513894498.meta
2024-11-13T16:04:54,511 WARN  [Thread-1056 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741908_1093
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:41321
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43616 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4]'}, localName='127.0.0.1:41913', datanodeUuid='02927f04-8187-4c41-87c8-0101e77f0f9c', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093 to mirror 127.0.0.1:41321
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,511 WARN  [Thread-1056 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:54,511 WARN  [Thread-1056 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093
2024-11-13T16:04:54,511 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43616 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093] {}] datanode.BlockReceiver(316): Block 1073741908 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:54,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43616 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741908_1093] {}] datanode.DataXceiver(331): 127.0.0.1:41913:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43616 dst: /127.0.0.1:41913
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,511 WARN  [Thread-1056 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:54,514 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,515 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,515 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
2024-11-13T16:04:54,515 WARN  [IPC Server handler 3 on default port 46231 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta has not been closed. Lease recovery is in progress. RecoveryId = 1095 for block blk_1073741834_1010
2024-11-13T16:04:54,515 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta after 0ms
2024-11-13T16:04:54,519 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36275:36275),(127.0.0.1/127.0.0.1:35673:35673)]
2024-11-13T16:04:54,519 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta is not closed yet, will try archiving it next time
2024-11-13T16:04:54,527 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741909_1094 (size=6027)
2024-11-13T16:04:54,528 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741909_1094 (size=6027)
2024-11-13T16:04:54,528 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.10 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f3e543082c4d4627853d9eae80567129
2024-11-13T16:04:54,536 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/.tmp/info/f3e543082c4d4627853d9eae80567129 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f3e543082c4d4627853d9eae80567129
2024-11-13T16:04:54,542 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/info/bb362438f9c64b39b5c7d3902303c46f is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6./info:regioninfo/1731513857600/Put/seqid=0
2024-11-13T16:04:54,543 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f3e543082c4d4627853d9eae80567129, entries=1, sequenceid=85, filesize=5.9 K
2024-11-13T16:04:54,544 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741910_1096
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,544 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741910_1096 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:54,544 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741910_1096
2024-11-13T16:04:54,544 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~2.10 KB/2150, heapSize ~2.48 KB/2544, currentSize=0 B/0 for 536ac2cf66251fda0ce168375295a1d6 in 48ms, sequenceid=85, compaction requested=false
2024-11-13T16:04:54,545 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:54,545 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7] to archive
2024-11-13T16:04:54,546 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741911_1097
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,546 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741911_1097 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:54,546 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741911_1097
2024-11-13T16:04:54,546 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:04:54,546 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:54,548 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4602593b92ba40f7b86cdcd6fbbde6db
2024-11-13T16:04:54,549 WARN  [Thread-1069 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741912_1098
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44501
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,549 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56310 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10]'}, localName='127.0.0.1:33387', datanodeUuid='a691b7c4-a458-43f8-a446-fceeebbdd418', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098 to mirror 127.0.0.1:44501
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,549 WARN  [Thread-1069 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK], DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:54,549 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56310 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098] {}] datanode.BlockReceiver(316): Block 1073741912 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:54,549 WARN  [Thread-1069 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098
2024-11-13T16:04:54,549 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:56310 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741912_1098] {}] datanode.DataXceiver(331): 127.0.0.1:33387:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56310 dst: /127.0.0.1:33387
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,550 WARN  [Thread-1069 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:54,550 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/eafec76df82440238b188c2ef28c479f
2024-11-13T16:04:54,551 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/7ce64e3d86ba4857a1f47b8541070328
2024-11-13T16:04:54,552 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/063a75c3472a4d15adee890cd42ed793
2024-11-13T16:04:54,554 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/b84ff9822df84143b12d75d56ec41415
2024-11-13T16:04:54,555 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c681067c3e34454793b6c6fd43de14c4
2024-11-13T16:04:54,557 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/f1b8866606d943e5bc2b73954c357252
2024-11-13T16:04:54,558 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/c42b05c24fc64144af3e313ded61df50
2024-11-13T16:04:54,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741913_1099 (size=7089)
2024-11-13T16:04:54,559 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741913_1099 (size=7089)
2024-11-13T16:04:54,560 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/info/bb362438f9c64b39b5c7d3902303c46f
2024-11-13T16:04:54,560 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/info/4e1793c571484b688c87741d8f5009e7
2024-11-13T16:04:54,561 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c4c650e80f58:40685 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-13T16:04:54,561 WARN  [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [4602593b92ba40f7b86cdcd6fbbde6db=10347, eafec76df82440238b188c2ef28c479f=12506, 7ce64e3d86ba4857a1f47b8541070328=17994, 063a75c3472a4d15adee890cd42ed793=6027, b84ff9822df84143b12d75d56ec41415=6027, c681067c3e34454793b6c6fd43de14c4=18097, f1b8866606d943e5bc2b73954c357252=6027, c42b05c24fc64144af3e313ded61df50=8190, 4e1793c571484b688c87741d8f5009e7=14660]
2024-11-13T16:04:54,566 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/default/TestLogRolling-testLogRollOnDatanodeDeath/536ac2cf66251fda0ce168375295a1d6/recovered.edits/88.seqid, newMaxSeqId=88, maxSeqId=1
2024-11-13T16:04:54,567 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:54,567 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 536ac2cf66251fda0ce168375295a1d6:
Waiting for close lock at 1731513894496Running coprocessor pre-close hooks at 1731513894496Disabling compacts and flushes for region at 1731513894496Disabling writes for close at 1731513894496Obtaining lock to block concurrent updates at 1731513894496Preparing flush snapshotting stores in 536ac2cf66251fda0ce168375295a1d6 at 1731513894496Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6., syncing WAL and waiting on mvcc, flushsize=dataSize=2150, getHeapSize=2544, getOffHeapSize=0, getCellsCount=2 at 1731513894497 (+1 ms)Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6. at 1731513894497Flushing 536ac2cf66251fda0ce168375295a1d6/info: creating writer at 1731513894498 (+1 ms)Flushing 536ac2cf66251fda0ce168375295a1d6/info: appending metadata at 1731513894501 (+3 ms)Flushing 536ac2cf66251fda0ce168375295a1d6/info: closing flushed file at 1731513894502 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3eff09da: reopening flushed file at 1731513894535 (+33 ms)Finished flush of dataSize ~2.10 KB/2150, heapSize ~2.48 KB/2544, currentSize=0 B/0 for 536ac2cf66251fda0ce168375295a1d6 in 48ms, sequenceid=85, compaction requested=false at 1731513894544 (+9 ms)Writing region close event to WAL at 1731513894562 (+18 ms)Running coprocessor post-close hooks at 1731513894567 (+5 ms)Closed at 1731513894567
2024-11-13T16:04:54,568 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731513857237.536ac2cf66251fda0ce168375295a1d6.
2024-11-13T16:04:54,583 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/ns/b1e693c81981412d9cbf15698674d1a5 is 43, key is default/ns:d/1731513856987/Put/seqid=0
2024-11-13T16:04:54,585 WARN  [Thread-1077 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741914_1100
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,586 WARN  [Thread-1077 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741914_1100 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]) is bad.
2024-11-13T16:04:54,586 WARN  [Thread-1077 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741914_1100
2024-11-13T16:04:54,586 WARN  [Thread-1077 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:41321,DS-46d503f4-6e4b-4d64-b58c-6ed37c4fd7dc,DISK]
2024-11-13T16:04:54,587 WARN  [Thread-1077 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741915_1101
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,588 WARN  [Thread-1077 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741915_1101 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK], DatanodeInfoWithStorage[127.0.0.1:33387,DS-d948f878-34d5-41dd-9ac2-1ec9a528e5f5,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]) is bad.
2024-11-13T16:04:54,588 WARN  [Thread-1077 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741915_1101
2024-11-13T16:04:54,588 WARN  [Thread-1077 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44501,DS-7af2de4f-57a8-48d3-9e33-2ea202a1986f,DISK]
2024-11-13T16:04:54,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741916_1102 (size=5153)
2024-11-13T16:04:54,593 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741916_1102 (size=5153)
2024-11-13T16:04:54,593 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/ns/b1e693c81981412d9cbf15698674d1a5
2024-11-13T16:04:54,615 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/table/7d64074e0117447ebb833c0639fc9c7f is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731513857611/Put/seqid=0
2024-11-13T16:04:54,618 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43638 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4]'}, localName='127.0.0.1:41913', datanodeUuid='02927f04-8187-4c41-87c8-0101e77f0f9c', xmitsInProgress=0}:Exception transferring block BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103 to mirror 127.0.0.1:45619
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,618 WARN  [Thread-1083 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741917_1103
java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45619
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:04:54,618 WARN  [Thread-1083 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41913,DS-31719626-8e9c-4b40-b0e8-d9156c01922a,DISK], DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]) is bad.
2024-11-13T16:04:54,618 WARN  [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43638 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103] {}] datanode.BlockReceiver(316): Block 1073741917 has not released the reserved bytes. Releasing 134217728 bytes as part of close.
2024-11-13T16:04:54,618 WARN  [Thread-1083 {}] hdfs.DataStreamer(1850): Abandoning BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103
2024-11-13T16:04:54,618 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_765153732_22 at /127.0.0.1:43638 [Receiving block BP-1920315224-172.17.0.3-1731513855204:blk_1073741917_1103] {}] datanode.DataXceiver(331): 127.0.0.1:41913:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:43638 dst: /127.0.0.1:41913
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:54,619 WARN  [Thread-1083 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45619,DS-1b775a48-ded5-4426-858c-d4c66c3dd3d3,DISK]
2024-11-13T16:04:54,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741918_1104 (size=5424)
2024-11-13T16:04:54,623 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741918_1104 (size=5424)
2024-11-13T16:04:54,624 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/table/7d64074e0117447ebb833c0639fc9c7f
2024-11-13T16:04:54,630 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/info/bb362438f9c64b39b5c7d3902303c46f as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/info/bb362438f9c64b39b5c7d3902303c46f
2024-11-13T16:04:54,635 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/info/bb362438f9c64b39b5c7d3902303c46f, entries=10, sequenceid=11, filesize=6.9 K
2024-11-13T16:04:54,636 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/ns/b1e693c81981412d9cbf15698674d1a5 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/ns/b1e693c81981412d9cbf15698674d1a5
2024-11-13T16:04:54,642 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/ns/b1e693c81981412d9cbf15698674d1a5, entries=2, sequenceid=11, filesize=5.0 K
2024-11-13T16:04:54,643 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/.tmp/table/7d64074e0117447ebb833c0639fc9c7f as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/table/7d64074e0117447ebb833c0639fc9c7f
2024-11-13T16:04:54,648 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/table/7d64074e0117447ebb833c0639fc9c7f, entries=2, sequenceid=11, filesize=5.3 K
2024-11-13T16:04:54,649 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 152ms, sequenceid=11, compaction requested=false
2024-11-13T16:04:54,654 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-13T16:04:54,655 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:54,655 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:54,655 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513894497Running coprocessor pre-close hooks at 1731513894497Disabling compacts and flushes for region at 1731513894497Disabling writes for close at 1731513894497Obtaining lock to block concurrent updates at 1731513894497Preparing flush snapshotting stores in 1588230740 at 1731513894497Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731513894498 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731513894520 (+22 ms)Flushing 1588230740/info: creating writer at 1731513894520Flushing 1588230740/info: appending metadata at 1731513894542 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731513894542Flushing 1588230740/ns: creating writer at 1731513894567 (+25 ms)Flushing 1588230740/ns: appending metadata at 1731513894583 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731513894583Flushing 1588230740/table: creating writer at 1731513894599 (+16 ms)Flushing 1588230740/table: appending metadata at 1731513894614 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731513894615 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4141b63b: reopening flushed file at 1731513894629 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@644cc23a: reopening flushed file at 1731513894635 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2add732b: reopening flushed file at 1731513894642 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 152ms, sequenceid=11, compaction requested=false at 1731513894649 (+7 ms)Writing region close event to WAL at 1731513894651 (+2 ms)Running coprocessor post-close hooks at 1731513894655 (+4 ms)Closed at 1731513894655
2024-11-13T16:04:54,656 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:04:54,697 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,41961,1731513856079; all regions closed.
2024-11-13T16:04:54,697 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,697 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,698 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,698 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,698 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:54,700 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741905_1090 (size=825)
2024-11-13T16:04:54,701 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741905_1090 (size=825)
2024-11-13T16:04:55,203 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:55,228 WARN  [java.util.concurrent.ThreadPoolExecutor$Worker@4c4b3609[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:33387, datanodeUuid=a691b7c4-a458-43f8-a446-fceeebbdd418, infoPort=35673, infoSecurePort=0, ipcPort=37433, storageInfo=lv=-57;cid=testClusterID;nsid=238867533;c=1731513855204):Failed to transfer BP-1920315224-172.17.0.3-1731513855204:blk_1073741875_1058 to 127.0.0.1:45619 got
java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:04:55,240 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-13T16:04:55,240 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-13T16:04:55,389 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-13T16:04:55,389 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-13T16:04:56,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath
2024-11-13T16:04:56,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:04:56,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:04:56,394 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:57,707 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-13T16:04:57,707 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-13T16:04:57,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741835_1011 (size=393)
2024-11-13T16:04:57,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:04:58,500 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 after 4002ms
2024-11-13T16:04:58,516 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta after 4001ms
2024-11-13T16:04:58,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:04:58,731 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:04:59,498 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-13T16:04:59,500 DEBUG [RS:1;c4c650e80f58:34133 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C34133%2C1731513857127:(num 1731513857337)
2024-11-13T16:04:59,500 DEBUG [RS:1;c4c650e80f58:34133 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:04:59,500 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:59,500 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:04:59,501 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:59,501 INFO  [RS:1;c4c650e80f58:34133 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34133
2024-11-13T16:04:59,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,34133,1731513857127
2024-11-13T16:04:59,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:59,503 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:59,505 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,34133,1731513857127]
2024-11-13T16:04:59,506 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87)
	at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77)
	at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:04:59,507 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,34133,1731513857127 already deleted, retry=false
2024-11-13T16:04:59,507 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,34133,1731513857127 expired; onlineServers=1
2024-11-13T16:04:59,568 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,584 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,585 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,591 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,592 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:04:59,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:59,605 INFO  [RS:1;c4c650e80f58:34133 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:59,605 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34133-0x100a608402a0002, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:59,605 INFO  [RS:1;c4c650e80f58:34133 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,34133,1731513857127; zookeeper connection closed.
2024-11-13T16:04:59,605 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67d152a1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67d152a1
2024-11-13T16:04:59,698 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds"
2024-11-13T16:04:59,702 DEBUG [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs
2024-11-13T16:04:59,702 INFO  [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C41961%2C1731513856079.meta:.meta(num 1731513894498)
2024-11-13T16:04:59,703 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,703 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,703 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,703 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,703 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,705 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741892_1076 (size=19324)
2024-11-13T16:04:59,706 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741892_1076 (size=19324)
2024-11-13T16:04:59,707 DEBUG [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C41961%2C1731513856079:(num 1731513893642)
2024-11-13T16:04:59,708 DEBUG [RS:0;c4c650e80f58:41961 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:59,708 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:04:59,708 INFO  [RS:0;c4c650e80f58:41961 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41961
2024-11-13T16:04:59,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,41961,1731513856079
2024-11-13T16:04:59,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:04:59,713 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:04:59,714 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,41961,1731513856079]
2024-11-13T16:04:59,717 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,41961,1731513856079 already deleted, retry=false
2024-11-13T16:04:59,717 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,41961,1731513856079 expired; onlineServers=0
2024-11-13T16:04:59,717 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,40685,1731513856028' *****
2024-11-13T16:04:59,717 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:04:59,717 INFO  [M:0;c4c650e80f58:40685 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:04:59,717 INFO  [M:0;c4c650e80f58:40685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:04:59,717 DEBUG [M:0;c4c650e80f58:40685 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:04:59,717 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:04:59,717 DEBUG [M:0;c4c650e80f58:40685 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:04:59,717 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513856278 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513856278,5,FailOnTimeoutGroup]
2024-11-13T16:04:59,717 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513856275 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513856275,5,FailOnTimeoutGroup]
2024-11-13T16:04:59,718 INFO  [M:0;c4c650e80f58:40685 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:04:59,718 INFO  [M:0;c4c650e80f58:40685 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:04:59,718 DEBUG [M:0;c4c650e80f58:40685 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:04:59,718 INFO  [M:0;c4c650e80f58:40685 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:04:59,718 INFO  [M:0;c4c650e80f58:40685 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:04:59,718 INFO  [M:0;c4c650e80f58:40685 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:04:59,718 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:04:59,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:04:59,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:04:59,720 DEBUG [M:0;c4c650e80f58:40685 {}] zookeeper.ZKUtil(347): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:04:59,720 WARN  [M:0;c4c650e80f58:40685 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:04:59,720 INFO  [M:0;c4c650e80f58:40685 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/.lastflushedseqids
2024-11-13T16:04:59,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741919_1105 (size=130)
2024-11-13T16:04:59,727 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741919_1105 (size=130)
2024-11-13T16:04:59,728 INFO  [M:0;c4c650e80f58:40685 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:04:59,728 INFO  [M:0;c4c650e80f58:40685 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:04:59,728 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:04:59,728 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:59,728 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:59,728 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:04:59,728 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:59,728 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB
2024-11-13T16:04:59,744 DEBUG [M:0;c4c650e80f58:40685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2239c13489014cd7851855bc916946a5 is 82, key is hbase:meta,,1/info:regioninfo/1731513856964/Put/seqid=0
2024-11-13T16:04:59,749 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741920_1106 (size=5672)
2024-11-13T16:04:59,750 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741920_1106 (size=5672)
2024-11-13T16:04:59,750 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2239c13489014cd7851855bc916946a5
2024-11-13T16:04:59,770 DEBUG [M:0;c4c650e80f58:40685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1d8d7fff37064187a2438650d1149a5f is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731513857615/Put/seqid=0
2024-11-13T16:04:59,775 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741921_1107 (size=6255)
2024-11-13T16:04:59,776 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741921_1107 (size=6255)
2024-11-13T16:04:59,777 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1d8d7fff37064187a2438650d1149a5f
2024-11-13T16:04:59,782 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d8d7fff37064187a2438650d1149a5f
2024-11-13T16:04:59,800 DEBUG [M:0;c4c650e80f58:40685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5487b06be3c44a27b50533ad54b912fb is 69, key is c4c650e80f58,34133,1731513857127/rs:state/1731513857184/Put/seqid=0
2024-11-13T16:04:59,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741922_1108 (size=5224)
2024-11-13T16:04:59,807 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741922_1108 (size=5224)
2024-11-13T16:04:59,808 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5487b06be3c44a27b50533ad54b912fb
2024-11-13T16:04:59,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:59,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41961-0x100a608402a0001, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:04:59,815 INFO  [RS:0;c4c650e80f58:41961 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:04:59,815 INFO  [RS:0;c4c650e80f58:41961 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,41961,1731513856079; zookeeper connection closed.
2024-11-13T16:04:59,815 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@34e3029c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@34e3029c
2024-11-13T16:04:59,815 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete
2024-11-13T16:04:59,828 DEBUG [M:0;c4c650e80f58:40685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/caa0fda1415740cfb31f7f0715df8459 is 52, key is load_balancer_on/state:d/1731513857110/Put/seqid=0
2024-11-13T16:04:59,833 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741923_1109 (size=5056)
2024-11-13T16:04:59,834 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741923_1109 (size=5056)
2024-11-13T16:04:59,834 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/caa0fda1415740cfb31f7f0715df8459
2024-11-13T16:04:59,841 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2239c13489014cd7851855bc916946a5 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2239c13489014cd7851855bc916946a5
2024-11-13T16:04:59,846 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2239c13489014cd7851855bc916946a5, entries=8, sequenceid=60, filesize=5.5 K
2024-11-13T16:04:59,847 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1d8d7fff37064187a2438650d1149a5f as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1d8d7fff37064187a2438650d1149a5f
2024-11-13T16:04:59,852 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d8d7fff37064187a2438650d1149a5f
2024-11-13T16:04:59,852 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1d8d7fff37064187a2438650d1149a5f, entries=6, sequenceid=60, filesize=6.1 K
2024-11-13T16:04:59,853 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5487b06be3c44a27b50533ad54b912fb as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5487b06be3c44a27b50533ad54b912fb
2024-11-13T16:04:59,859 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5487b06be3c44a27b50533ad54b912fb, entries=2, sequenceid=60, filesize=5.1 K
2024-11-13T16:04:59,860 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/caa0fda1415740cfb31f7f0715df8459 as hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/caa0fda1415740cfb31f7f0715df8459
2024-11-13T16:04:59,866 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/caa0fda1415740cfb31f7f0715df8459, entries=1, sequenceid=60, filesize=4.9 K
2024-11-13T16:04:59,867 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false
2024-11-13T16:04:59,869 INFO  [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:04:59,869 DEBUG [M:0;c4c650e80f58:40685 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513899728Disabling compacts and flushes for region at 1731513899728Disabling writes for close at 1731513899728Obtaining lock to block concurrent updates at 1731513899728Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731513899728Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731513899729 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731513899729Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731513899730 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731513899744 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731513899744Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731513899755 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731513899770 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731513899770Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731513899783 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731513899799 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731513899799Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731513899813 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731513899828 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731513899828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d3523cd: reopening flushed file at 1731513899840 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20055e39: reopening flushed file at 1731513899846 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41c305a7: reopening flushed file at 1731513899853 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76b65977: reopening flushed file at 1731513899859 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false at 1731513899867 (+8 ms)Writing region close event to WAL at 1731513899868 (+1 ms)Closed at 1731513899868
2024-11-13T16:04:59,869 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,869 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,869 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,869 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,870 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:04:59,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741889_1072 (size=1045)
2024-11-13T16:04:59,872 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41913 is added to blk_1073741889_1072 (size=1045)
2024-11-13T16:05:00,094 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:05:00,108 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,109 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,110 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,113 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,116 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:00,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:00,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:00,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:05:00,730 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33387 is added to blk_1073741836_1012 (size=76)
2024-11-13T16:05:00,748 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@61d53049 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:45619,null,null])
java.net.ConnectException: Call From c4c650e80f58/172.17.0.3 to localhost:37367 failed on connection exception: java.net.ConnectException: Connection refused; For more details see:  http://wiki.apache.org/hadoop/ConnectionRefused
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.net.ConnectException: Connection refused
	at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?]
	at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-13T16:05:01,301 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/WALs/c4c650e80f58,40685,1731513856028/c4c650e80f58%2C40685%2C1731513856028.1731513856186 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/oldWALs/c4c650e80f58%2C40685%2C1731513856028.1731513856186
2024-11-13T16:05:01,305 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/MasterData/oldWALs/c4c650e80f58%2C40685%2C1731513856028.1731513856186 to hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/oldWALs/c4c650e80f58%2C40685%2C1731513856028.1731513856186$masterlocalwal$
2024-11-13T16:05:01,305 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:05:01,305 INFO  [M:0;c4c650e80f58:40685 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:05:01,305 INFO  [M:0;c4c650e80f58:40685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40685
2024-11-13T16:05:01,306 INFO  [M:0;c4c650e80f58:40685 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:05:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:01,408 INFO  [M:0;c4c650e80f58:40685 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:05:01,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40685-0x100a608402a0000, quorum=127.0.0.1:49300, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:01,410 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@117da513{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:01,410 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6842affb{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:01,410 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:01,411 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@61815e22{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:01,411 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@236c96ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:01,412 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:01,412 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:01,412 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:01,412 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid 02927f04-8187-4c41-87c8-0101e77f0f9c) service to localhost/127.0.0.1:46231
2024-11-13T16:05:01,412 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:45619,null,null])
java.io.InterruptedIOException: DestHost:destPort localhost:37367 , LocalHost:localPort c4c650e80f58/172.17.0.3:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS)
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
Caused by: java.lang.InterruptedException: sleep interrupted
	at java.lang.Thread.sleep(Native Method) ~[?:?]
	at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?]
	... 12 more
2024-11-13T16:05:01,413 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data3/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:01,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:41913,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1920315224-172.17.0.3-1731513855204
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:01,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45619,null,null], DatanodeInfoWithStorage[127.0.0.1:41913,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:45619,null,null], DatanodeInfoWithStorage[127.0.0.1:41913,null,null]]
2024-11-13T16:05:01,413 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data4/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:01,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45619,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1920315224-172.17.0.3-1731513855204
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:01,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:41913,null,null])
java.io.IOException: No block pool offer service for bpid=BP-1920315224-172.17.0.3-1731513855204
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:01,413 WARN  [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@19d34839 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45619,null,null], DatanodeInfoWithStorage[127.0.0.1:41913,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1920315224-172.17.0.3-1731513855204:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:45619,null,null], DatanodeInfoWithStorage[127.0.0.1:41913,null,null]]
2024-11-13T16:05:01,413 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:01,415 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7d285dba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:01,416 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@158a9d8c{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:01,416 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:01,416 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@687696f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:01,416 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e96eece{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:01,417 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:01,417 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:01,417 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:01,417 WARN  [BP-1920315224-172.17.0.3-1731513855204 heartbeating to localhost/127.0.0.1:46231 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1920315224-172.17.0.3-1731513855204 (Datanode Uuid a691b7c4-a458-43f8-a446-fceeebbdd418) service to localhost/127.0.0.1:46231
2024-11-13T16:05:01,418 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data9/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:01,418 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/cluster_cbb3d104-21f5-d5c6-e4b0-408108652233/data/data10/current/BP-1920315224-172.17.0.3-1731513855204 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:01,418 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:01,424 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43ade8e8{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:05:01,424 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ac253d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:01,425 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:01,425 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6de86657{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:01,425 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67cf8368{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:01,433 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:05:01,477 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:05:01,487 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=153 (was 78)
Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: ForkJoinPool-2-worker-5
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724)
	java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623)
	java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:46231
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:37687
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:46231
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-20-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46231
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46231 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46231
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37687
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-5-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:46231
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46231
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46231 from jenkins.hfs.2
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-21-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46231
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-7-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6d5cbf4db8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-18-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Timer for 'DataNode' metrics system
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563)
	java.base@17.0.11/java.util.TimerThread.run(Timer.java:516)

Potentially hanging thread: HMaster-EventLoopGroup-5-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46231 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-19-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-21-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: Close-WAL-Writer-0
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175)
	app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044)
	app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f6d5cbf4db8.run(Unknown Source)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46231 from jenkins.hfs.3
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)
 - Thread LEAK? -, OpenFileDescriptor=452 (was 404) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=192 (was 123) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11098 (was 11561)
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=153, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=192, ProcessCount=11, AvailableMemoryMB=11096
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.log.dir so I do NOT create it in target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/61bae153-ac83-e5cc-75fc-c328c9899285/hadoop.tmp.dir so I do NOT create it in target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab, deleteOnExit=true
2024-11-13T16:05:01,496 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/test.cache.data in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:05:01,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:05:01,497 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:05:01,498 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:05:01,507 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:01,512 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:05:01,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:01,575 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:01,579 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:01,588 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:01,588 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:01,588 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:01,589 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:01,590 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25a29a07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:01,591 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7096145a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:01,706 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@49ef22be{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-46289-hadoop-hdfs-3_4_1-tests_jar-_-any-8473411707392398255/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:05:01,707 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1b230242{HTTP/1.1, (http/1.1)}{localhost:46289}
2024-11-13T16:05:01,707 INFO  [Time-limited test {}] server.Server(415): Started @150593ms
2024-11-13T16:05:01,720 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:05:01,783 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:01,786 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:01,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:01,788 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:01,788 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:01,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1fc8bed8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:01,789 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13646a74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:01,903 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5fb42f9a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-45711-hadoop-hdfs-3_4_1-tests_jar-_-any-18001304765844661300/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:01,904 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d74671d{HTTP/1.1, (http/1.1)}{localhost:45711}
2024-11-13T16:05:01,904 INFO  [Time-limited test {}] server.Server(415): Started @150790ms
2024-11-13T16:05:01,906 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:01,936 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:01,939 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:01,940 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:01,940 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:01,940 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:01,941 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64434c96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:01,941 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47946b20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:02,012 WARN  [Thread-1209 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data2/current/BP-48576801-172.17.0.3-1731513901520/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:02,012 WARN  [Thread-1208 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data1/current/BP-48576801-172.17.0.3-1731513901520/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:02,030 WARN  [Thread-1187 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:02,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdbb4f384fddfaaf0 with lease ID 0x766177cc86c76f50: Processing first storage report for DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f from datanode DatanodeRegistration(127.0.0.1:35015, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=42053, infoSecurePort=0, ipcPort=45163, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520)
2024-11-13T16:05:02,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbb4f384fddfaaf0 with lease ID 0x766177cc86c76f50: from storage DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f node DatanodeRegistration(127.0.0.1:35015, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=42053, infoSecurePort=0, ipcPort=45163, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:02,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdbb4f384fddfaaf0 with lease ID 0x766177cc86c76f50: Processing first storage report for DS-12ab0480-bab6-41a7-a2cd-c1f145095ee4 from datanode DatanodeRegistration(127.0.0.1:35015, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=42053, infoSecurePort=0, ipcPort=45163, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520)
2024-11-13T16:05:02,032 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdbb4f384fddfaaf0 with lease ID 0x766177cc86c76f50: from storage DS-12ab0480-bab6-41a7-a2cd-c1f145095ee4 node DatanodeRegistration(127.0.0.1:35015, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=42053, infoSecurePort=0, ipcPort=45163, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:02,059 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@37687418{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-36391-hadoop-hdfs-3_4_1-tests_jar-_-any-17979067199803124873/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:02,059 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e7f214b{HTTP/1.1, (http/1.1)}{localhost:36391}
2024-11-13T16:05:02,059 INFO  [Time-limited test {}] server.Server(415): Started @150945ms
2024-11-13T16:05:02,060 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:02,162 WARN  [Thread-1234 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data3/current/BP-48576801-172.17.0.3-1731513901520/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:02,162 WARN  [Thread-1235 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data4/current/BP-48576801-172.17.0.3-1731513901520/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:02,186 WARN  [Thread-1223 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:02,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1caf5c2a2438c65b with lease ID 0x766177cc86c76f51: Processing first storage report for DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf from datanode DatanodeRegistration(127.0.0.1:41577, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40879, infoSecurePort=0, ipcPort=45583, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520)
2024-11-13T16:05:02,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1caf5c2a2438c65b with lease ID 0x766177cc86c76f51: from storage DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf node DatanodeRegistration(127.0.0.1:41577, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40879, infoSecurePort=0, ipcPort=45583, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:02,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1caf5c2a2438c65b with lease ID 0x766177cc86c76f51: Processing first storage report for DS-691684e2-2fcc-4f43-a9fb-a0b6357eb5e0 from datanode DatanodeRegistration(127.0.0.1:41577, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40879, infoSecurePort=0, ipcPort=45583, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520)
2024-11-13T16:05:02,188 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1caf5c2a2438c65b with lease ID 0x766177cc86c76f51: from storage DS-691684e2-2fcc-4f43-a9fb-a0b6357eb5e0 node DatanodeRegistration(127.0.0.1:41577, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40879, infoSecurePort=0, ipcPort=45583, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:02,289 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a
2024-11-13T16:05:02,292 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/zookeeper_0, clientPort=57849, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:05:02,293 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57849
2024-11-13T16:05:02,293 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,294 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,303 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:05:02,304 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:05:02,305 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4 with version=8
2024-11-13T16:05:02,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:05:02,307 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:05:02,308 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:05:02,308 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37965
2024-11-13T16:05:02,310 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37965 connecting to ZooKeeper ensemble=127.0.0.1:57849
2024-11-13T16:05:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379650x0, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:05:02,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37965-0x100a608f4f30000 connected
2024-11-13T16:05:02,340 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,341 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:02,343 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4, hbase.cluster.distributed=false
2024-11-13T16:05:02,345 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:05:02,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37965
2024-11-13T16:05:02,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37965
2024-11-13T16:05:02,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37965
2024-11-13T16:05:02,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37965
2024-11-13T16:05:02,349 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37965
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:05:02,365 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:05:02,366 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44501
2024-11-13T16:05:02,367 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44501 connecting to ZooKeeper ensemble=127.0.0.1:57849
2024-11-13T16:05:02,368 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,370 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445010x0, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:05:02,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445010x0, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:02,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44501-0x100a608f4f30001 connected
2024-11-13T16:05:02,375 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:05:02,378 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:05:02,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:05:02,380 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:05:02,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44501
2024-11-13T16:05:02,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44501
2024-11-13T16:05:02,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44501
2024-11-13T16:05:02,389 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44501
2024-11-13T16:05:02,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44501
2024-11-13T16:05:02,404 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:37965
2024-11-13T16:05:02,405 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:02,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:02,407 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:05:02,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,412 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:05:02,413 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,37965,1731513902307 from backup master directory
2024-11-13T16:05:02,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:02,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:02,414 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:05:02,414 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,419 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/hbase.id] with ID: 28bb6b65-10cc-4cb2-8940-0e8f9de8defa
2024-11-13T16:05:02,419 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/.tmp/hbase.id
2024-11-13T16:05:02,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:05:02,429 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:05:02,429 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/.tmp/hbase.id]:[hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/hbase.id]
2024-11-13T16:05:02,441 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:02,442 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:05:02,443 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-13T16:05:02,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:05:02,453 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:05:02,454 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:05:02,455 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:05:02,455 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:02,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:05:02,463 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:05:02,464 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store
2024-11-13T16:05:02,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:05:02,471 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:05:02,471 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:02,472 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:05:02,472 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:02,472 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:02,472 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:05:02,472 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:02,472 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:02,472 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513902471Disabling compacts and flushes for region at 1731513902471Disabling writes for close at 1731513902472 (+1 ms)Writing region close event to WAL at 1731513902472Closed at 1731513902472
2024-11-13T16:05:02,473 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/.initializing
2024-11-13T16:05:02,473 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,475 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C37965%2C1731513902307, suffix=, logDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307, archiveDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/oldWALs, maxLogs=10
2024-11-13T16:05:02,475 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C37965%2C1731513902307.1731513902475
2024-11-13T16:05:02,482 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475
2024-11-13T16:05:02,486 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40879:40879),(127.0.0.1/127.0.0.1:42053:42053)]
2024-11-13T16:05:02,487 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:02,488 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:02,488 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,488 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,490 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:02,492 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:05:02,492 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,493 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:02,493 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:02,494 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:05:02,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,494 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:02,494 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:02,495 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:05:02,496 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,496 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:02,496 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:02,497 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:05:02,497 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,498 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:02,498 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,498 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,499 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,500 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,500 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,500 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:05:02,501 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:02,503 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:02,504 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=710849, jitterRate=-0.09610918164253235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:05:02,504 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513902488Initializing all the Stores at 1731513902489 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513902489Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513902490 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513902490Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513902490Cleaning up temporary data from old regions at 1731513902500 (+10 ms)Region opened successfully at 1731513902504 (+4 ms)
2024-11-13T16:05:02,505 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:05:02,508 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708d8792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:05:02,508 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:02,509 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:05:02,509 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:05:02,509 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:05:02,509 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:05:02,509 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:05:02,510 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:05:02,510 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:05:02,512 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:05:02,513 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:05:02,514 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:05:02,515 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:05:02,515 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:05:02,518 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:05:02,518 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:05:02,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:02,519 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:05:02,521 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:05:02,522 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:05:02,523 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:05:02,525 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:05:02,526 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:05:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,530 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,37965,1731513902307, sessionid=0x100a608f4f30000, setting cluster-up flag (Was=false)
2024-11-13T16:05:02,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,539 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:05:02,540 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:02,548 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:05:02,549 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,37965,1731513902307
2024-11-13T16:05:02,550 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:05:02,552 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:02,552 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:05:02,553 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:05:02,553 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,37965,1731513902307
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:05:02,554 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,555 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731513932555
2024-11-13T16:05:02,555 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:05:02,555 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:02,556 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:05:02,556 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:05:02,557 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513902557,5,FailOnTimeoutGroup]
2024-11-13T16:05:02,557 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,557 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513902557,5,FailOnTimeoutGroup]
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:05:02,557 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,557 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:05:02,568 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:05:02,569 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:05:02,569 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4
2024-11-13T16:05:02,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:05:02,576 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:05:02,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:02,578 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:05:02,579 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:05:02,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:02,580 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:05:02,581 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:05:02,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,582 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:02,582 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:05:02,583 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:05:02,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,584 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:02,584 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:05:02,585 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:05:02,585 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:02,586 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:02,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:05:02,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740
2024-11-13T16:05:02,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740
2024-11-13T16:05:02,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:05:02,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:05:02,589 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:05:02,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:05:02,592 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:02,592 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751384, jitterRate=-0.0445658415555954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:05:02,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513902577Initializing all the Stores at 1731513902577Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513902578 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513902578Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513902578Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513902578Cleaning up temporary data from old regions at 1731513902588 (+10 ms)Region opened successfully at 1731513902593 (+5 ms)
2024-11-13T16:05:02,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:05:02,593 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:05:02,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:05:02,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:05:02,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:05:02,594 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(746): ClusterId : 28bb6b65-10cc-4cb2-8940-0e8f9de8defa
2024-11-13T16:05:02,594 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:05:02,594 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:05:02,594 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513902593Disabling compacts and flushes for region at 1731513902593Disabling writes for close at 1731513902593Writing region close event to WAL at 1731513902594 (+1 ms)Closed at 1731513902594
2024-11-13T16:05:02,595 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:02,595 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:05:02,595 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:05:02,596 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:05:02,596 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:05:02,597 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:05:02,598 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:05:02,600 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:05:02,600 DEBUG [RS:0;c4c650e80f58:44501 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c086a92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:05:02,612 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:44501
2024-11-13T16:05:02,612 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:05:02,612 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:05:02,612 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:05:02,613 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,37965,1731513902307 with port=44501, startcode=1731513902365
2024-11-13T16:05:02,614 DEBUG [RS:0;c4c650e80f58:44501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:05:02,616 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50597, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:05:02,616 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37965 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,616 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37965 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,618 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4
2024-11-13T16:05:02,618 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40901
2024-11-13T16:05:02,618 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:05:02,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:05:02,620 DEBUG [RS:0;c4c650e80f58:44501 {}] zookeeper.ZKUtil(111): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,621 WARN  [RS:0;c4c650e80f58:44501 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:05:02,621 INFO  [RS:0;c4c650e80f58:44501 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:02,621 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,621 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,44501,1731513902365]
2024-11-13T16:05:02,624 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:05:02,626 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:05:02,630 INFO  [RS:0;c4c650e80f58:44501 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:05:02,630 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,631 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:05:02,632 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:05:02,632 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,632 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:05:02,633 DEBUG [RS:0;c4c650e80f58:44501 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:05:02,634 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,634 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,634 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,634 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,635 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,635 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,44501,1731513902365-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:05:02,650 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:05:02,651 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,44501,1731513902365-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,651 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,651 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.Replication(171): c4c650e80f58,44501,1731513902365 started
2024-11-13T16:05:02,665 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:02,665 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,44501,1731513902365, RpcServer on c4c650e80f58/172.17.0.3:44501, sessionid=0x100a608f4f30001
2024-11-13T16:05:02,665 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:05:02,665 DEBUG [RS:0;c4c650e80f58:44501 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,665 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,44501,1731513902365'
2024-11-13T16:05:02,665 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,44501,1731513902365
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,44501,1731513902365'
2024-11-13T16:05:02,666 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:05:02,667 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:05:02,667 DEBUG [RS:0;c4c650e80f58:44501 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:05:02,667 INFO  [RS:0;c4c650e80f58:44501 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:05:02,667 INFO  [RS:0;c4c650e80f58:44501 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:05:02,748 WARN  [c4c650e80f58:37965 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:05:02,769 INFO  [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C44501%2C1731513902365, suffix=, logDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365, archiveDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs, maxLogs=32
2024-11-13T16:05:02,770 INFO  [RS:0;c4c650e80f58:44501 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:02,776 INFO  [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:02,777 DEBUG [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40879:40879),(127.0.0.1/127.0.0.1:42053:42053)]
2024-11-13T16:05:02,999 DEBUG [c4c650e80f58:37965 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:05:02,999 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:03,001 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,44501,1731513902365, state=OPENING
2024-11-13T16:05:03,002 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:05:03,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:03,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:03,004 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:05:03,004 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,44501,1731513902365}]
2024-11-13T16:05:03,005 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:03,005 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:03,158 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:05:03,160 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54995, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:05:03,163 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:05:03,164 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:03,165 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C44501%2C1731513902365.meta, suffix=.meta, logDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365, archiveDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs, maxLogs=32
2024-11-13T16:05:03,166 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta
2024-11-13T16:05:03,171 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta
2024-11-13T16:05:03,172 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42053:42053),(127.0.0.1/127.0.0.1:40879:40879)]
2024-11-13T16:05:03,173 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:03,173 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:05:03,173 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:05:03,173 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:05:03,174 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:05:03,174 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:03,174 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:05:03,174 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:05:03,175 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:05:03,176 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:05:03,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:03,176 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:05:03,177 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:05:03,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:03,178 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:05:03,178 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:05:03,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,179 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:03,179 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:05:03,179 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:05:03,180 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,180 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:03,180 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:05:03,181 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740
2024-11-13T16:05:03,182 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740
2024-11-13T16:05:03,183 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:05:03,183 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:05:03,184 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:05:03,185 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:05:03,186 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737470, jitterRate=-0.06225892901420593}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:05:03,186 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:05:03,187 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513903174Writing region info on filesystem at 1731513903174Initializing all the Stores at 1731513903175 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513903175Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513903175Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513903175Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513903175Cleaning up temporary data from old regions at 1731513903183 (+8 ms)Running coprocessor post-open hooks at 1731513903186 (+3 ms)Region opened successfully at 1731513903187 (+1 ms)
2024-11-13T16:05:03,188 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513903157
2024-11-13T16:05:03,190 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:05:03,190 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:05:03,191 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:03,192 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,44501,1731513902365, state=OPEN
2024-11-13T16:05:03,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:05:03,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:05:03,197 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:03,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:03,197 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:03,200 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:05:03,200 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,44501,1731513902365 in 193 msec
2024-11-13T16:05:03,203 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:05:03,203 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec
2024-11-13T16:05:03,204 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:03,204 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:05:03,205 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:05:03,206 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,44501,1731513902365, seqNum=-1]
2024-11-13T16:05:03,206 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:05:03,207 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39643, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:05:03,213 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 660 msec
2024-11-13T16:05:03,213 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513903213, completionTime=-1
2024-11-13T16:05:03,213 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:05:03,213 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731513963215
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514023215
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:37965, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,215 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,216 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,217 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:05:03,219 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.805sec
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:05:03,220 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:05:03,222 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:05:03,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:05:03,222 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,37965,1731513902307-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:03,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6beed7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:03,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,37965,-1 for getting cluster id
2024-11-13T16:05:03,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:05:03,296 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '28bb6b65-10cc-4cb2-8940-0e8f9de8defa'
2024-11-13T16:05:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:05:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "28bb6b65-10cc-4cb2-8940-0e8f9de8defa"

2024-11-13T16:05:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52af6006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,37965,-1]
2024-11-13T16:05:03,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:05:03,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:03,299 INFO  [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:05:03,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61726e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:03,300 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:05:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,44501,1731513902365, seqNum=-1]
2024-11-13T16:05:03,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:05:03,303 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:05:03,305 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,37965,1731513902307
2024-11-13T16:05:03,305 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:03,307 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:05:03,307 INFO  [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart
2024-11-13T16:05:03,307 INFO  [Time-limited test {}] wal.TestLogRolling(323): Replication=2
2024-11-13T16:05:03,308 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-13T16:05:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is c4c650e80f58,37965,1731513902307
2024-11-13T16:05:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2887615f
2024-11-13T16:05:03,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-13T16:05:03,310 INFO  [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-13T16:05:03,311 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-13T16:05:03,311 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-13T16:05:03,311 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:05:03,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart
2024-11-13T16:05:03,314 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION
2024-11-13T16:05:03,314 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,314 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRollOnPipelineRestart"
 procId is: 4
2024-11-13T16:05:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:05:03,315 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-13T16:05:03,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741835_1011 (size=395)
2024-11-13T16:05:03,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741835_1011 (size=395)
2024-11-13T16:05:03,324 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f62bc58ab0104f7ecca734a2dcaac3f1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4
2024-11-13T16:05:03,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35015 is added to blk_1073741836_1012 (size=78)
2024-11-13T16:05:03,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41577 is added to blk_1073741836_1012 (size=78)
2024-11-13T16:05:03,331 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:03,331 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing f62bc58ab0104f7ecca734a2dcaac3f1, disabling compactions & flushes
2024-11-13T16:05:03,331 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,331 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,331 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1. after waiting 0 ms
2024-11-13T16:05:03,331 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,331 INFO  [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,332 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for f62bc58ab0104f7ecca734a2dcaac3f1:
Waiting for close lock at 1731513903331Disabling compacts and flushes for region at 1731513903331Disabling writes for close at 1731513903331Writing region close event to WAL at 1731513903331Closed at 1731513903331
2024-11-13T16:05:03,333 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META
2024-11-13T16:05:03,334 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731513903333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731513903333"}]},"ts":"1731513903333"}
2024-11-13T16:05:03,337 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-13T16:05:03,338 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-13T16:05:03,339 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513903338"}]},"ts":"1731513903338"}
2024-11-13T16:05:03,341 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta
2024-11-13T16:05:03,341 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f62bc58ab0104f7ecca734a2dcaac3f1, ASSIGN}]
2024-11-13T16:05:03,342 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f62bc58ab0104f7ecca734a2dcaac3f1, ASSIGN
2024-11-13T16:05:03,343 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f62bc58ab0104f7ecca734a2dcaac3f1, ASSIGN; state=OFFLINE, location=c4c650e80f58,44501,1731513902365; forceNewPlan=false, retain=false
2024-11-13T16:05:03,494 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f62bc58ab0104f7ecca734a2dcaac3f1, regionState=OPENING, regionLocation=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:03,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f62bc58ab0104f7ecca734a2dcaac3f1, ASSIGN because future has completed
2024-11-13T16:05:03,497 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f62bc58ab0104f7ecca734a2dcaac3f1, server=c4c650e80f58,44501,1731513902365}]
2024-11-13T16:05:03,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:03,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:03,655 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,655 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f62bc58ab0104f7ecca734a2dcaac3f1, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:03,655 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,655 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:03,655 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,655 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,657 INFO  [StoreOpener-f62bc58ab0104f7ecca734a2dcaac3f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f62bc58ab0104f7ecca734a2dcaac3f1 
2024-11-13T16:05:03,658 INFO  [StoreOpener-f62bc58ab0104f7ecca734a2dcaac3f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f62bc58ab0104f7ecca734a2dcaac3f1 columnFamilyName info
2024-11-13T16:05:03,658 DEBUG [StoreOpener-f62bc58ab0104f7ecca734a2dcaac3f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:03,658 INFO  [StoreOpener-f62bc58ab0104f7ecca734a2dcaac3f1-1 {}] regionserver.HStore(327): Store=f62bc58ab0104f7ecca734a2dcaac3f1/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:03,659 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,659 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,660 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,660 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,660 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,662 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,664 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:03,665 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f62bc58ab0104f7ecca734a2dcaac3f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=769666, jitterRate=-0.02131977677345276}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:05:03,665 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:03,666 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f62bc58ab0104f7ecca734a2dcaac3f1:
Running coprocessor pre-open hook at 1731513903655Writing region info on filesystem at 1731513903655Initializing all the Stores at 1731513903656 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513903656Cleaning up temporary data from old regions at 1731513903660 (+4 ms)Running coprocessor post-open hooks at 1731513903665 (+5 ms)Region opened successfully at 1731513903666 (+1 ms)
2024-11-13T16:05:03,667 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1., pid=6, masterSystemTime=1731513903651
2024-11-13T16:05:03,670 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,670 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:03,671 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f62bc58ab0104f7ecca734a2dcaac3f1, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,44501,1731513902365
2024-11-13T16:05:03,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f62bc58ab0104f7ecca734a2dcaac3f1, server=c4c650e80f58,44501,1731513902365 because future has completed
2024-11-13T16:05:03,677 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-13T16:05:03,677 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f62bc58ab0104f7ecca734a2dcaac3f1, server=c4c650e80f58,44501,1731513902365 in 178 msec
2024-11-13T16:05:03,680 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-13T16:05:03,680 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=f62bc58ab0104f7ecca734a2dcaac3f1, ASSIGN in 336 msec
2024-11-13T16:05:03,681 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-13T16:05:03,681 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513903681"}]},"ts":"1731513903681"}
2024-11-13T16:05:03,684 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta
2024-11-13T16:05:03,685 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION
2024-11-13T16:05:03,687 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 374 msec
2024-11-13T16:05:04,509 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:04,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:05,510 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:05,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:06,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:05:06,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-13T16:05:06,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-13T16:05:06,371 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer
2024-11-13T16:05:06,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:05:06,372 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-13T16:05:06,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:06,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:07,511 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:07,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:08,512 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:08,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:08,677 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:05:08,703 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,704 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,704 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,705 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,705 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,706 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,711 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,716 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:08,725 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-13T16:05:08,726 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart'
2024-11-13T16:05:09,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:09,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:10,513 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:10,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:11,514 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:11,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:12,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:12,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37965 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:05:13,404 INFO  [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed
2024-11-13T16:05:13,405 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100
2024-11-13T16:05:13,408 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart
2024-11-13T16:05:13,408 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:13,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1., hostname=c4c650e80f58,44501,1731513902365, seqNum=2]
2024-11-13T16:05:13,515 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:13,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:14,516 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:14,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:15,417 INFO  [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:15,417 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,417 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010
java.io.IOException: Bad response ERROR for BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,417 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,418 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK], DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]) is bad.
2024-11-13T16:05:15,418 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK], DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]) is bad.
2024-11-13T16:05:15,418 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK], DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41577,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]) is bad.
2024-11-13T16:05:15,418 WARN  [PacketResponder: BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41577] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,418 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:49460 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:41577:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49460 dst: /127.0.0.1:41577
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:60400 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60400 dst: /127.0.0.1:35015
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1492647045_22 at /127.0.0.1:49444 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:41577:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49444 dst: /127.0.0.1:41577
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:60402 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60402 dst: /127.0.0.1:35015
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:49470 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:41577:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:49470 dst: /127.0.0.1:41577
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,419 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1492647045_22 at /127.0.0.1:60378 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:60378 dst: /127.0.0.1:35015
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,421 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@37687418{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:15,422 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e7f214b{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:15,422 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:15,422 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47946b20{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:15,422 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64434c96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:15,423 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:15,423 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:15,423 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid a7046ce0-40bc-409d-9428-bc3f3ca072ed) service to localhost/127.0.0.1:40901
2024-11-13T16:05:15,423 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:15,424 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data3/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:15,424 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data4/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:15,424 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:15,432 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:15,435 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:15,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:15,436 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:15,436 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:15,437 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2460467f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:15,437 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75f58649{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:15,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:15,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:15,554 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c5aa216{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-36165-hadoop-hdfs-3_4_1-tests_jar-_-any-11668755691889379595/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:15,554 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@46332285{HTTP/1.1, (http/1.1)}{localhost:36165}
2024-11-13T16:05:15,554 INFO  [Time-limited test {}] server.Server(415): Started @164440ms
2024-11-13T16:05:15,555 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:15,573 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1013
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,573 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1015
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,573 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1014
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:15,574 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:54462 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54462 dst: /127.0.0.1:35015
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,574 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1492647045_22 at /127.0.0.1:54476 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54476 dst: /127.0.0.1:35015
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,574 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:54474 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35015:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54474 dst: /127.0.0.1:35015
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:15,579 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5fb42f9a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:15,580 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d74671d{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:15,580 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:15,580 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13646a74{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:15,580 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1fc8bed8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:15,581 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:15,581 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid 3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2) service to localhost/127.0.0.1:40901
2024-11-13T16:05:15,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data2/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:15,582 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:15,582 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data1/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:15,582 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:15,582 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:15,623 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:15,626 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:15,627 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:15,627 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:15,627 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:05:15,627 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75ed142f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:15,628 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ede944f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:15,691 WARN  [Thread-1358 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:15,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbca474f432c0a5b7 with lease ID 0x766177cc86c76f52: from storage DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf node DatanodeRegistration(127.0.0.1:39021, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40523, infoSecurePort=0, ipcPort=35735, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:05:15,695 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbca474f432c0a5b7 with lease ID 0x766177cc86c76f52: from storage DS-691684e2-2fcc-4f43-a9fb-a0b6357eb5e0 node DatanodeRegistration(127.0.0.1:39021, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=40523, infoSecurePort=0, ipcPort=35735, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:15,751 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7180ac25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-39415-hadoop-hdfs-3_4_1-tests_jar-_-any-4509317270830064604/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:15,751 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@248d0d6a{HTTP/1.1, (http/1.1)}{localhost:39415}
2024-11-13T16:05:15,751 INFO  [Time-limited test {}] server.Server(415): Started @164637ms
2024-11-13T16:05:15,753 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:15,855 WARN  [Thread-1389 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:15,857 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf8486f8e5f7a80d with lease ID 0x766177cc86c76f53: from storage DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f node DatanodeRegistration(127.0.0.1:41489, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=40357, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:05:15,858 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdf8486f8e5f7a80d with lease ID 0x766177cc86c76f53: from storage DS-12ab0480-bab6-41a7-a2cd-c1f145095ee4 node DatanodeRegistration(127.0.0.1:41489, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=40357, infoSecurePort=0, ipcPort=40717, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:16,517 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:16,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:16,773 INFO  [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted
2024-11-13T16:05:16,776 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002
2024-11-13T16:05:16,777 ERROR [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:16,777 WARN  [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:16,777 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C44501%2C1731513902365:(num 1731513902770) roll requested
2024-11-13T16:05:16,778 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:16,784 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 newFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:16,784 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:16,784 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:16,784 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:16,784 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:16,784 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:16,784 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:16,785 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:16,785 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:16,785 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:16,785 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40523:40523),(127.0.0.1/127.0.0.1:40357:40357)]
2024-11-13T16:05:16,785 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 is not closed yet, will try archiving it next time
2024-11-13T16:05:16,785 WARN  [IPC Server handler 3 on default port 40901 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013
2024-11-13T16:05:16,786 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 after 1ms
2024-11-13T16:05:16,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41489 is added to blk_1073741833_1017 (size=1632)
2024-11-13T16:05:17,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:17,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:18,518 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:18,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:18,789 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003
2024-11-13T16:05:19,519 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:19,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:20,520 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:20,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:20,694 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-13T16:05:20,787 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 after 4001ms
2024-11-13T16:05:20,792 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016
java.io.IOException: Bad response ERROR for BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:41489,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:20,793 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39021,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK], DatanodeInfoWithStorage[127.0.0.1:41489,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:41489,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]) is bad.
2024-11-13T16:05:20,793 WARN  [PacketResponder: BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:41489] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): 
java.io.IOException: Connection reset by peer
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
	at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?]
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?]
	at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?]
	at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?]
	at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?]
	at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?]
	at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:20,793 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:54622 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:54622 dst: /127.0.0.1:39021
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:20,793 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:34628 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41489:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:34628 dst: /127.0.0.1:41489
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:20,795 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7180ac25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:20,795 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@248d0d6a{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:20,795 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:20,796 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ede944f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:20,796 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75ed142f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:20,798 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:20,798 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:20,798 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:20,798 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid 3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2) service to localhost/127.0.0.1:40901
2024-11-13T16:05:20,799 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data1/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:20,799 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data2/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:20,800 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:20,809 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:20,813 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:20,814 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:20,815 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:20,815 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:05:20,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@19016e01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:20,818 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ef50a45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:20,968 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6db938{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-39701-hadoop-hdfs-3_4_1-tests_jar-_-any-5305153715751009641/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:20,969 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5cf288ef{HTTP/1.1, (http/1.1)}{localhost:39701}
2024-11-13T16:05:20,969 INFO  [Time-limited test {}] server.Server(415): Started @169854ms
2024-11-13T16:05:20,970 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:20,994 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1018
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:20,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1925556451_22 at /127.0.0.1:41182 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39021:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:41182 dst: /127.0.0.1:39021
java.nio.channels.ClosedChannelException: null
	at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?]
	at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?]
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:21,000 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c5aa216{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:21,000 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@46332285{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:21,000 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:21,001 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75f58649{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:21,001 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2460467f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:21,002 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:21,002 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:21,002 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:21,002 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid a7046ce0-40bc-409d-9428-bc3f3ca072ed) service to localhost/127.0.0.1:40901
2024-11-13T16:05:21,002 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data3/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:21,003 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data4/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:21,003 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:21,021 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:21,024 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:21,028 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:21,028 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:21,028 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:21,030 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c39138a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:21,031 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5f3e5a16{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:21,076 WARN  [Thread-1432 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:21,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16de9f4c669394ae with lease ID 0x766177cc86c76f54: from storage DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f node DatanodeRegistration(127.0.0.1:40143, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=41239, infoSecurePort=0, ipcPort=44711, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:21,079 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16de9f4c669394ae with lease ID 0x766177cc86c76f54: from storage DS-12ab0480-bab6-41a7-a2cd-c1f145095ee4 node DatanodeRegistration(127.0.0.1:40143, datanodeUuid=3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2, infoPort=41239, infoSecurePort=0, ipcPort=44711, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:21,161 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@41b9791e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/java.io.tmpdir/jetty-localhost-34919-hadoop-hdfs-3_4_1-tests_jar-_-any-3847869310904226531/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:21,162 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1240a3cc{HTTP/1.1, (http/1.1)}{localhost:34919}
2024-11-13T16:05:21,162 INFO  [Time-limited test {}] server.Server(415): Started @170048ms
2024-11-13T16:05:21,164 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:21,289 WARN  [Thread-1463 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:21,294 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe211cb18b5fa05da with lease ID 0x766177cc86c76f55: from storage DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf node DatanodeRegistration(127.0.0.1:41701, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=44095, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:21,294 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe211cb18b5fa05da with lease ID 0x766177cc86c76f55: from storage DS-691684e2-2fcc-4f43-a9fb-a0b6357eb5e0 node DatanodeRegistration(127.0.0.1:41701, datanodeUuid=a7046ce0-40bc-409d-9428-bc3f3ca072ed, infoPort=44095, infoSecurePort=0, ipcPort=34235, storageInfo=lv=-57;cid=testClusterID;nsid=443762518;c=1731513901520), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:21,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:21,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:22,192 INFO  [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted
2024-11-13T16:05:22,195 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004
2024-11-13T16:05:22,196 ERROR [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39021,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:22,196 WARN  [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39021,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:22,196 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C44501%2C1731513902365:(num 1731513916778) roll requested
2024-11-13T16:05:22,196 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:22,203 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 newFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:22,203 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:22,203 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:22,203 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:22,203 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:22,204 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:22,204 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:22,204 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39021,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:22,204 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:39021,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:22,204 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:22,204 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41239:41239),(127.0.0.1/127.0.0.1:44095:44095)]
2024-11-13T16:05:22,205 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 is not closed yet, will try archiving it next time
2024-11-13T16:05:22,205 WARN  [IPC Server handler 4 on default port 40901 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018
2024-11-13T16:05:22,205 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 after 1ms
2024-11-13T16:05:22,294 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741837_1020 (size=2427)
2024-11-13T16:05:22,521 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:22,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:23,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:23,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:24,206 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:24,212 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196 newFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:24,212 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:24,212 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:24,212 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:24,212 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:24,212 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:24,212 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:24,213 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44095:44095),(127.0.0.1/127.0.0.1:41239:41239)]
2024-11-13T16:05:24,213 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 is not closed yet, will try archiving it next time
2024-11-13T16:05:24,214 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196 is not closed yet, will try archiving it next time
2024-11-13T16:05:24,214 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:24,214 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:24,214 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741838_1019 (size=1264)
2024-11-13T16:05:24,215 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741838_1019 (size=1264)
2024-11-13T16:05:24,215 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770 after 1ms
2024-11-13T16:05:24,215 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:24,215 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 is not closed yet, will try archiving it next time
2024-11-13T16:05:24,227 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731513903666/Put/vlen=218/seqid=0]
2024-11-13T16:05:24,227 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731513913415/Put/vlen=1045/seqid=0]
2024-11-13T16:05:24,227 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513902770
2024-11-13T16:05:24,227 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:24,228 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:24,228 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 after 0ms
2024-11-13T16:05:24,228 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:24,232 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731513916777/Put/vlen=1045/seqid=0]
2024-11-13T16:05:24,232 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731513918790/Put/vlen=1045/seqid=0]
2024-11-13T16:05:24,232 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778
2024-11-13T16:05:24,232 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:24,232 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:24,233 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196 after 1ms
2024-11-13T16:05:24,233 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513922196
2024-11-13T16:05:24,237 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731513922196/Put/vlen=1045/seqid=0]
2024-11-13T16:05:24,237 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:24,237 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:24,237 WARN  [IPC Server handler 0 on default port 40901 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021
2024-11-13T16:05:24,238 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 after 1ms
2024-11-13T16:05:24,522 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:24,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:25,082 WARN  [ResponseProcessor for block BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021
java.io.EOFException: Unexpected EOF while trying to read response from server
	at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:25,082 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1492647045_22 at /127.0.0.1:56616 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:40143:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:56616 dst: /127.0.0.1:40143
java.io.IOException: Premature EOF from inputStream
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:25,082 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1492647045_22 at /127.0.0.1:37002 [Receiving block BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41701:DataXceiver error processing WRITE_BLOCK operation  src: /127.0.0.1:37002 dst: /127.0.0.1:41701
java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41701 remote=/127.0.0.1:37002]. Total timeout mills is 60000, 59129 millis timeout left.
	at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?]
	at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?]
	at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?]
	at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?]
	at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?]
	at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:05:25,083 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 block BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41701,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK], DatanodeInfoWithStorage[127.0.0.1:40143,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41701,DS-d213ca8b-79f6-4bf2-8065-659c840d2cbf,DISK]) is bad.
2024-11-13T16:05:25,084 WARN  [DataStreamer for file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 block BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:25,088 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741839_1022 (size=85)
2024-11-13T16:05:25,523 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:25,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:26,080 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-13T16:05:26,206 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513916778 after 4002ms
2024-11-13T16:05:26,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:26,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:27,524 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:27,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:28,239 INFO  [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 after 4002ms
2024-11-13T16:05:28,239 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:28,243 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:28,243 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing f62bc58ab0104f7ecca734a2dcaac3f1 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB
2024-11-13T16:05:28,244 ERROR [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,244 WARN  [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365 {}] wal.AbstractFSWAL(2174): append entry failed
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,244 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C44501%2C1731513902365:(num 1731513924206) roll requested
2024-11-13T16:05:28,245 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.1731513928244
2024-11-13T16:05:28,251 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 newFile=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513928244
2024-11-13T16:05:28,251 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,251 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,251 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,251 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,251 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,252 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513928244
2024-11-13T16:05:28,252 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,252 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-48576801-172.17.0.3-1731513901520:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912)
	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980)
	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002)
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182)
	at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589)
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573)
	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246)
	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169)
	at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
	at java.base/javax.security.auth.Subject.doAs(Subject.java:439)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953)
	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198)

	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?]
	at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?]
	at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?]
	at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,253 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:28,253 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 after 0ms
2024-11-13T16:05:28,256 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.1731513924206 to hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs/c4c650e80f58%2C44501%2C1731513902365.1731513924206
2024-11-13T16:05:28,256 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41239:41239),(127.0.0.1/127.0.0.1:44095:44095)]
2024-11-13T16:05:28,273 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/.tmp/info/f81f1bdadd414e8b87b942cb68532022 is 1080, key is row1002/info:/1731513913415/Put/seqid=0
2024-11-13T16:05:28,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741841_1024 (size=9270)
2024-11-13T16:05:28,281 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741841_1024 (size=9270)
2024-11-13T16:05:28,283 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/.tmp/info/f81f1bdadd414e8b87b942cb68532022
2024-11-13T16:05:28,290 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/.tmp/info/f81f1bdadd414e8b87b942cb68532022 as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/info/f81f1bdadd414e8b87b942cb68532022
2024-11-13T16:05:28,295 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/info/f81f1bdadd414e8b87b942cb68532022, entries=4, sequenceid=8, filesize=9.1 K
2024-11-13T16:05:28,296 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for f62bc58ab0104f7ecca734a2dcaac3f1 in 53ms, sequenceid=8, compaction requested=false
2024-11-13T16:05:28,296 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f62bc58ab0104f7ecca734a2dcaac3f1:

2024-11-13T16:05:28,297 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB
2024-11-13T16:05:28,297 ERROR [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,297 WARN  [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4-prefix:c4c650e80f58,44501,1731513902365.meta {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,297 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C44501%2C1731513902365.meta:.meta(num 1731513903166) roll requested
2024-11-13T16:05:28,297 INFO  [regionserver/c4c650e80f58:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44501%2C1731513902365.meta.1731513928297.meta
2024-11-13T16:05:28,302 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,303 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,303 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,303 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,303 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,303 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513928297.meta
2024-11-13T16:05:28,303 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,304 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:28,304 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta
2024-11-13T16:05:28,304 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41239:41239),(127.0.0.1/127.0.0.1:44095:44095)]
2024-11-13T16:05:28,304 DEBUG [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta is not closed yet, will try archiving it next time
2024-11-13T16:05:28,304 WARN  [IPC Server handler 3 on default port 40901 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta has not been closed. Lease recovery is in progress. RecoveryId = 1026 for block blk_1073741834_1014
2024-11-13T16:05:28,304 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta after 0ms
2024-11-13T16:05:28,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/info/4dfa69573ee84d9d8a8f83fd9e79abd0 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1./info:regioninfo/1731513903671/Put/seqid=0
2024-11-13T16:05:28,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741843_1027 (size=7125)
2024-11-13T16:05:28,327 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741843_1027 (size=7125)
2024-11-13T16:05:28,327 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/info/4dfa69573ee84d9d8a8f83fd9e79abd0
2024-11-13T16:05:28,349 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/ns/fb8cbadaea4a402bb543ef4df0468cec is 43, key is default/ns:d/1731513903208/Put/seqid=0
2024-11-13T16:05:28,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741844_1028 (size=5153)
2024-11-13T16:05:28,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741844_1028 (size=5153)
2024-11-13T16:05:28,356 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/ns/fb8cbadaea4a402bb543ef4df0468cec
2024-11-13T16:05:28,377 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/table/0f0774ebf50f43ccba7ba195f13de6f6 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731513903681/Put/seqid=0
2024-11-13T16:05:28,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741845_1029 (size=5438)
2024-11-13T16:05:28,382 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741845_1029 (size=5438)
2024-11-13T16:05:28,382 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/table/0f0774ebf50f43ccba7ba195f13de6f6
2024-11-13T16:05:28,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/info/4dfa69573ee84d9d8a8f83fd9e79abd0 as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/info/4dfa69573ee84d9d8a8f83fd9e79abd0
2024-11-13T16:05:28,393 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/info/4dfa69573ee84d9d8a8f83fd9e79abd0, entries=10, sequenceid=11, filesize=7.0 K
2024-11-13T16:05:28,394 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/ns/fb8cbadaea4a402bb543ef4df0468cec as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/ns/fb8cbadaea4a402bb543ef4df0468cec
2024-11-13T16:05:28,399 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/ns/fb8cbadaea4a402bb543ef4df0468cec, entries=2, sequenceid=11, filesize=5.0 K
2024-11-13T16:05:28,400 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/.tmp/table/0f0774ebf50f43ccba7ba195f13de6f6 as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/table/0f0774ebf50f43ccba7ba195f13de6f6
2024-11-13T16:05:28,405 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/table/0f0774ebf50f43ccba7ba195f13de6f6, entries=2, sequenceid=11, filesize=5.3 K
2024-11-13T16:05:28,406 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=11, compaction requested=false
2024-11-13T16:05:28,406 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-13T16:05:28,411 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:05:28,411 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:05:28,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:05:28,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:28,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:28,412 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:05:28,412 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:05:28,412 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=191438578, stopped=false
2024-11-13T16:05:28,412 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,37965,1731513902307
2024-11-13T16:05:28,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:28,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:28,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:28,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:28,414 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:05:28,414 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:05:28,414 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:05:28,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:28,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:28,415 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,44501,1731513902365' *****
2024-11-13T16:05:28,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:28,415 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:05:28,415 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:05:28,415 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:05:28,415 INFO  [RS:0;c4c650e80f58:44501 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:05:28,415 INFO  [RS:0;c4c650e80f58:44501 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:05:28,415 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(3091): Received CLOSE for f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,44501,1731513902365
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:44501.
2024-11-13T16:05:28,416 DEBUG [RS:0;c4c650e80f58:44501 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:05:28,416 DEBUG [RS:0;c4c650e80f58:44501 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f62bc58ab0104f7ecca734a2dcaac3f1, disabling compactions & flushes
2024-11-13T16:05:28,416 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1. after waiting 0 ms
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:28,416 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-13T16:05:28,416 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1325): Online Regions={f62bc58ab0104f7ecca734a2dcaac3f1=TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1., 1588230740=hbase:meta,,1.1588230740}
2024-11-13T16:05:28,416 DEBUG [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f62bc58ab0104f7ecca734a2dcaac3f1
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:05:28,416 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:05:28,416 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:05:28,421 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/default/TestLogRolling-testLogRollOnPipelineRestart/f62bc58ab0104f7ecca734a2dcaac3f1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1
2024-11-13T16:05:28,421 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-13T16:05:28,422 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:28,422 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f62bc58ab0104f7ecca734a2dcaac3f1:
Waiting for close lock at 1731513928416Running coprocessor pre-close hooks at 1731513928416Disabling compacts and flushes for region at 1731513928416Disabling writes for close at 1731513928416Writing region close event to WAL at 1731513928417 (+1 ms)Running coprocessor post-close hooks at 1731513928422 (+5 ms)Closed at 1731513928422
2024-11-13T16:05:28,422 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:05:28,422 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:05:28,422 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513928416Running coprocessor pre-close hooks at 1731513928416Disabling compacts and flushes for region at 1731513928416Disabling writes for close at 1731513928416Writing region close event to WAL at 1731513928418 (+2 ms)Running coprocessor post-close hooks at 1731513928422 (+4 ms)Closed at 1731513928422
2024-11-13T16:05:28,422 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731513903311.f62bc58ab0104f7ecca734a2dcaac3f1.
2024-11-13T16:05:28,422 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:05:28,525 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:28,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:28,616 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,44501,1731513902365; all regions closed.
2024-11-13T16:05:28,617 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,617 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,617 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,617 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,618 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:28,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741842_1025 (size=825)
2024-11-13T16:05:28,620 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741842_1025 (size=825)
2024-11-13T16:05:28,635 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped
2024-11-13T16:05:28,635 INFO  [regionserver/c4c650e80f58:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped
2024-11-13T16:05:28,637 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:05:29,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:29,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:30,526 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:30,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:31,527 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:31,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:32,289 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:05:32,294 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-13T16:05:32,305 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta after 4001ms
2024-11-13T16:05:32,305 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/WALs/c4c650e80f58,44501,1731513902365/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta to hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs/c4c650e80f58%2C44501%2C1731513902365.meta.1731513903166.meta
2024-11-13T16:05:32,308 DEBUG [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs
2024-11-13T16:05:32,308 INFO  [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C44501%2C1731513902365.meta:.meta(num 1731513928297)
2024-11-13T16:05:32,309 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,309 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,309 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,309 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,309 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741840_1023 (size=1162)
2024-11-13T16:05:32,311 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741840_1023 (size=1162)
2024-11-13T16:05:32,315 DEBUG [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs
2024-11-13T16:05:32,315 INFO  [RS:0;c4c650e80f58:44501 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C44501%2C1731513902365:(num 1731513928244)
2024-11-13T16:05:32,315 DEBUG [RS:0;c4c650e80f58:44501 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:32,315 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:05:32,315 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:05:32,315 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-13T16:05:32,316 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:05:32,316 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:05:32,316 INFO  [RS:0;c4c650e80f58:44501 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44501
2024-11-13T16:05:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,44501,1731513902365
2024-11-13T16:05:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:05:32,318 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:05:32,321 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,44501,1731513902365]
2024-11-13T16:05:32,322 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,44501,1731513902365 already deleted, retry=false
2024-11-13T16:05:32,322 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,44501,1731513902365 expired; onlineServers=0
2024-11-13T16:05:32,322 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,37965,1731513902307' *****
2024-11-13T16:05:32,322 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:05:32,322 INFO  [M:0;c4c650e80f58:37965 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:05:32,322 INFO  [M:0;c4c650e80f58:37965 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:05:32,322 DEBUG [M:0;c4c650e80f58:37965 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:05:32,322 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:05:32,322 DEBUG [M:0;c4c650e80f58:37965 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:05:32,322 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513902557 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513902557,5,FailOnTimeoutGroup]
2024-11-13T16:05:32,322 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513902557 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513902557,5,FailOnTimeoutGroup]
2024-11-13T16:05:32,323 INFO  [M:0;c4c650e80f58:37965 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:05:32,323 INFO  [M:0;c4c650e80f58:37965 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:05:32,323 DEBUG [M:0;c4c650e80f58:37965 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:05:32,323 INFO  [M:0;c4c650e80f58:37965 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:05:32,323 INFO  [M:0;c4c650e80f58:37965 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:05:32,323 INFO  [M:0;c4c650e80f58:37965 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:05:32,323 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:05:32,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:05:32,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:32,324 DEBUG [M:0;c4c650e80f58:37965 {}] zookeeper.ZKUtil(347): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:05:32,324 WARN  [M:0;c4c650e80f58:37965 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:05:32,325 INFO  [M:0;c4c650e80f58:37965 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/.lastflushedseqids
2024-11-13T16:05:32,330 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741846_1030 (size=130)
2024-11-13T16:05:32,331 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741846_1030 (size=130)
2024-11-13T16:05:32,331 INFO  [M:0;c4c650e80f58:37965 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:05:32,331 INFO  [M:0;c4c650e80f58:37965 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:05:32,331 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:05:32,331 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:32,331 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:32,331 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:05:32,331 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:32,331 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.16 KB heapSize=29.13 KB
2024-11-13T16:05:32,332 ERROR [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData-prefix:c4c650e80f58,37965,1731513902307 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:32,332 WARN  [FSHLog-0-hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData-prefix:c4c650e80f58,37965,1731513902307 {}] wal.AbstractFSWAL(2174): append entry failed
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:32,332 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c4c650e80f58%2C37965%2C1731513902307:(num 1731513902475) roll requested
2024-11-13T16:05:32,332 INFO  [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C37965%2C1731513902307.1731513932332
2024-11-13T16:05:32,337 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,337 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,338 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,338 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,338 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,338 INFO  [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 with entries=53, filesize=26.61 KB; new WAL /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513932332
2024-11-13T16:05:32,338 WARN  [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing...
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:32,338 WARN  [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed.
java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:35015,DS-4bf55e24-84c9-403f-9bf7-23ef7ba6516f,DISK]] are bad. Aborting...
	at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?]
2024-11-13T16:05:32,338 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475
2024-11-13T16:05:32,339 WARN  [IPC Server handler 1 on default port 40901 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015
2024-11-13T16:05:32,339 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 after 1ms
2024-11-13T16:05:32,342 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41239:41239),(127.0.0.1/127.0.0.1:44095:44095)]
2024-11-13T16:05:32,342 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 is not closed yet, will try archiving it next time
2024-11-13T16:05:32,358 DEBUG [M:0;c4c650e80f58:37965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/969aa9be0be7422586a7edcab4aa4ec7 is 82, key is hbase:meta,,1/info:regioninfo/1731513903191/Put/seqid=0
2024-11-13T16:05:32,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741848_1033 (size=5672)
2024-11-13T16:05:32,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741848_1033 (size=5672)
2024-11-13T16:05:32,365 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/969aa9be0be7422586a7edcab4aa4ec7
2024-11-13T16:05:32,385 DEBUG [M:0;c4c650e80f58:37965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e981afa6e447468e0729f6dd8e3faf is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731513903686/Put/seqid=0
2024-11-13T16:05:32,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741849_1034 (size=6117)
2024-11-13T16:05:32,390 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741849_1034 (size=6117)
2024-11-13T16:05:32,391 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.56 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e981afa6e447468e0729f6dd8e3faf
2024-11-13T16:05:32,417 DEBUG [M:0;c4c650e80f58:37965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f8a994071894a08a267526af51ff05f is 69, key is c4c650e80f58,44501,1731513902365/rs:state/1731513902617/Put/seqid=0
2024-11-13T16:05:32,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:32,421 INFO  [RS:0;c4c650e80f58:44501 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:05:32,421 INFO  [RS:0;c4c650e80f58:44501 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,44501,1731513902365; zookeeper connection closed.
2024-11-13T16:05:32,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44501-0x100a608f4f30001, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:32,421 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3d41c2c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3d41c2c9
2024-11-13T16:05:32,421 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:05:32,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741850_1035 (size=5156)
2024-11-13T16:05:32,423 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741850_1035 (size=5156)
2024-11-13T16:05:32,424 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f8a994071894a08a267526af51ff05f
2024-11-13T16:05:32,445 DEBUG [M:0;c4c650e80f58:37965 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/911804dcc8e548cfb0d8753c4580e0c7 is 52, key is load_balancer_on/state:d/1731513903306/Put/seqid=0
2024-11-13T16:05:32,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741851_1036 (size=5056)
2024-11-13T16:05:32,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741851_1036 (size=5056)
2024-11-13T16:05:32,450 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/911804dcc8e548cfb0d8753c4580e0c7
2024-11-13T16:05:32,456 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/969aa9be0be7422586a7edcab4aa4ec7 as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/969aa9be0be7422586a7edcab4aa4ec7
2024-11-13T16:05:32,460 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/969aa9be0be7422586a7edcab4aa4ec7, entries=8, sequenceid=56, filesize=5.5 K
2024-11-13T16:05:32,461 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/09e981afa6e447468e0729f6dd8e3faf as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09e981afa6e447468e0729f6dd8e3faf
2024-11-13T16:05:32,466 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/09e981afa6e447468e0729f6dd8e3faf, entries=6, sequenceid=56, filesize=6.0 K
2024-11-13T16:05:32,467 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f8a994071894a08a267526af51ff05f as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f8a994071894a08a267526af51ff05f
2024-11-13T16:05:32,471 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f8a994071894a08a267526af51ff05f, entries=1, sequenceid=56, filesize=5.0 K
2024-11-13T16:05:32,472 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/911804dcc8e548cfb0d8753c4580e0c7 as hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/911804dcc8e548cfb0d8753c4580e0c7
2024-11-13T16:05:32,476 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/911804dcc8e548cfb0d8753c4580e0c7, entries=1, sequenceid=56, filesize=4.9 K
2024-11-13T16:05:32,477 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=56, compaction requested=false
2024-11-13T16:05:32,479 INFO  [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:32,479 DEBUG [M:0;c4c650e80f58:37965 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513932331Disabling compacts and flushes for region at 1731513932331Disabling writes for close at 1731513932331Obtaining lock to block concurrent updates at 1731513932331Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731513932331Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23714, getHeapSize=29768, getOffHeapSize=0, getCellsCount=67 at 1731513932332 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731513932343 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731513932343Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731513932358 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731513932358Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731513932370 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731513932384 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731513932384Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731513932395 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731513932417 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731513932417Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731513932429 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731513932444 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731513932444Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bdd80f4: reopening flushed file at 1731513932455 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eaf83b6: reopening flushed file at 1731513932461 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32b8b2b7: reopening flushed file at 1731513932466 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d63545: reopening flushed file at 1731513932471 (+5 ms)Finished flush of dataSize ~23.16 KB/23714, heapSize ~29.07 KB/29768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=56, compaction requested=false at 1731513932477 (+6 ms)Writing region close event to WAL at 1731513932479 (+2 ms)Closed at 1731513932479
2024-11-13T16:05:32,479 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,480 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,480 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,480 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,480 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:05:32,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741847_1031 (size=757)
2024-11-13T16:05:32,482 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40143 is added to blk_1073741847_1031 (size=757)
2024-11-13T16:05:32,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:32,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:33,422 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,423 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,445 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,446 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,446 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,446 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,449 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,449 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,449 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,452 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,458 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,459 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,528 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:33,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:33,962 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:05:33,963 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,963 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,963 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,964 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,985 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,986 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,987 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,991 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,992 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:33,995 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:34,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:34,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:35,294 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command
java.io.IOException: Failed to delete 1 (out of 1) replica(s):
0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?]
2024-11-13T16:05:35,529 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:35,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:36,340 INFO  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 after 4002ms
2024-11-13T16:05:36,340 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/WALs/c4c650e80f58,37965,1731513902307/c4c650e80f58%2C37965%2C1731513902307.1731513902475 to hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/oldWALs/c4c650e80f58%2C37965%2C1731513902307.1731513902475
2024-11-13T16:05:36,343 INFO  [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/MasterData/oldWALs/c4c650e80f58%2C37965%2C1731513902307.1731513902475 to hdfs://localhost:40901/user/jenkins/test-data/fcbeda83-300c-c977-ce26-bb5cb37c73e4/oldWALs/c4c650e80f58%2C37965%2C1731513902307.1731513902475$masterlocalwal$
2024-11-13T16:05:36,343 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:05:36,343 INFO  [M:0;c4c650e80f58:37965 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:05:36,343 INFO  [M:0;c4c650e80f58:37965 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37965
2024-11-13T16:05:36,344 INFO  [M:0;c4c650e80f58:37965 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:05:36,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:05:36,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:05:36,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-13T16:05:36,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart
2024-11-13T16:05:36,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:36,445 INFO  [M:0;c4c650e80f58:37965 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:05:36,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37965-0x100a608f4f30000, quorum=127.0.0.1:57849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:05:36,448 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@41b9791e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:36,448 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1240a3cc{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:36,448 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:36,449 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5f3e5a16{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:36,449 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c39138a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:36,450 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:36,450 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:36,450 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid a7046ce0-40bc-409d-9428-bc3f3ca072ed) service to localhost/127.0.0.1:40901
2024-11-13T16:05:36,450 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:36,451 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data3/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:36,451 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data4/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:36,451 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:36,456 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6db938{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:36,457 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5cf288ef{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:36,457 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:36,457 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ef50a45{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:36,457 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@19016e01{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:36,458 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:05:36,458 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:05:36,458 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:05:36,458 WARN  [BP-48576801-172.17.0.3-1731513901520 heartbeating to localhost/127.0.0.1:40901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-48576801-172.17.0.3-1731513901520 (Datanode Uuid 3c4eba0e-b5b6-44b9-999d-fadf98c9b7c2) service to localhost/127.0.0.1:40901
2024-11-13T16:05:36,459 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data1/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:36,459 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/cluster_a2543da8-4d90-2f28-7ebe-5c6f606b78ab/data/data2/current/BP-48576801-172.17.0.3-1731513901520 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:05:36,459 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:05:36,465 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@49ef22be{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:05:36,466 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1b230242{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:05:36,466 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:05:36,466 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7096145a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:05:36,466 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25a29a07{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir/,STOPPED}
2024-11-13T16:05:36,472 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:05:36,489 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:05:36,500 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 153)
Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40901 from jenkins.hfs.4
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-33-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40901 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-31-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:40901 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:40901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:40901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-10-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-33-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-30-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-32-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-31-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=457 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 192), ProcessCount=11 (was 11), AvailableMemoryMB=11464 (was 11096) - AvailableMemoryMB LEAK? -
2024-11-13T16:05:36,509 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=11464
2024-11-13T16:05:36,509 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.log.dir so I do NOT create it in target/test-data/b5200649-de5a-5913-b985-fe441a4fa661
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/8eaa9102-86e8-f20d-ee29-1fe6a63eea9a/hadoop.tmp.dir so I do NOT create it in target/test-data/b5200649-de5a-5913-b985-fe441a4fa661
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9, deleteOnExit=true
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/test.cache.data in system properties and HBase conf
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:05:36,510 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:05:36,510 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:05:36,511 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:05:36,525 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:05:36,530 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:36,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:36,598 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:36,601 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:36,602 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:36,602 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:36,602 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:05:36,603 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:36,603 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@510b5e02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:36,604 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d6dee42{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:36,717 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ecaedd6{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/java.io.tmpdir/jetty-localhost-36375-hadoop-hdfs-3_4_1-tests_jar-_-any-14982293171126130494/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:05:36,718 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16aeea80{HTTP/1.1, (http/1.1)}{localhost:36375}
2024-11-13T16:05:36,718 INFO  [Time-limited test {}] server.Server(415): Started @185603ms
2024-11-13T16:05:36,730 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:05:36,807 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:36,811 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:36,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:36,813 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:36,813 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:05:36,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dd1f450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:36,814 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4e873b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:36,937 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bc64a6a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/java.io.tmpdir/jetty-localhost-36049-hadoop-hdfs-3_4_1-tests_jar-_-any-3406695421532423698/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:36,938 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49fb27f0{HTTP/1.1, (http/1.1)}{localhost:36049}
2024-11-13T16:05:36,938 INFO  [Time-limited test {}] server.Server(415): Started @185824ms
2024-11-13T16:05:36,939 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:36,971 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:05:36,974 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:05:36,975 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:05:36,975 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:05:36,975 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:05:36,975 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e5fe848{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:05:36,975 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30873421{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:05:37,058 WARN  [Thread-1657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data1/current/BP-607491404-172.17.0.3-1731513936542/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:37,058 WARN  [Thread-1658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data2/current/BP-607491404-172.17.0.3-1731513936542/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:37,082 WARN  [Thread-1636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:37,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb6e1a7474d1abe1f with lease ID 0x40627a3ae962ecf1: Processing first storage report for DS-f6d7d01e-7258-4dc0-a4b1-33c27bba94f0 from datanode DatanodeRegistration(127.0.0.1:46833, datanodeUuid=48ca5872-ccbd-4932-989e-0b81f04c498a, infoPort=39099, infoSecurePort=0, ipcPort=37301, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542)
2024-11-13T16:05:37,084 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6e1a7474d1abe1f with lease ID 0x40627a3ae962ecf1: from storage DS-f6d7d01e-7258-4dc0-a4b1-33c27bba94f0 node DatanodeRegistration(127.0.0.1:46833, datanodeUuid=48ca5872-ccbd-4932-989e-0b81f04c498a, infoPort=39099, infoSecurePort=0, ipcPort=37301, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0
2024-11-13T16:05:37,085 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb6e1a7474d1abe1f with lease ID 0x40627a3ae962ecf1: Processing first storage report for DS-fedc32fd-20a5-4da7-b0c3-73efce5b9205 from datanode DatanodeRegistration(127.0.0.1:46833, datanodeUuid=48ca5872-ccbd-4932-989e-0b81f04c498a, infoPort=39099, infoSecurePort=0, ipcPort=37301, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542)
2024-11-13T16:05:37,085 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb6e1a7474d1abe1f with lease ID 0x40627a3ae962ecf1: from storage DS-fedc32fd-20a5-4da7-b0c3-73efce5b9205 node DatanodeRegistration(127.0.0.1:46833, datanodeUuid=48ca5872-ccbd-4932-989e-0b81f04c498a, infoPort=39099, infoSecurePort=0, ipcPort=37301, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:37,106 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5daea1f2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/java.io.tmpdir/jetty-localhost-38495-hadoop-hdfs-3_4_1-tests_jar-_-any-5330283049205111605/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:05:37,106 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@448bc78{HTTP/1.1, (http/1.1)}{localhost:38495}
2024-11-13T16:05:37,106 INFO  [Time-limited test {}] server.Server(415): Started @185992ms
2024-11-13T16:05:37,107 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:05:37,203 WARN  [Thread-1683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data3/current/BP-607491404-172.17.0.3-1731513936542/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:37,203 WARN  [Thread-1684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data4/current/BP-607491404-172.17.0.3-1731513936542/current, will proceed with Du for space computation calculation, 
2024-11-13T16:05:37,226 WARN  [Thread-1672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:05:37,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86014881431998af with lease ID 0x40627a3ae962ecf2: Processing first storage report for DS-df864f77-3c3d-4511-9fdf-a42959245f64 from datanode DatanodeRegistration(127.0.0.1:35079, datanodeUuid=c979ebad-dd51-47f8-91dc-d332f576cf02, infoPort=46661, infoSecurePort=0, ipcPort=41389, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542)
2024-11-13T16:05:37,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86014881431998af with lease ID 0x40627a3ae962ecf2: from storage DS-df864f77-3c3d-4511-9fdf-a42959245f64 node DatanodeRegistration(127.0.0.1:35079, datanodeUuid=c979ebad-dd51-47f8-91dc-d332f576cf02, infoPort=46661, infoSecurePort=0, ipcPort=41389, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:37,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x86014881431998af with lease ID 0x40627a3ae962ecf2: Processing first storage report for DS-2558a032-9d13-4146-a1de-ee7b3c4d9049 from datanode DatanodeRegistration(127.0.0.1:35079, datanodeUuid=c979ebad-dd51-47f8-91dc-d332f576cf02, infoPort=46661, infoSecurePort=0, ipcPort=41389, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542)
2024-11-13T16:05:37,228 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x86014881431998af with lease ID 0x40627a3ae962ecf2: from storage DS-2558a032-9d13-4146-a1de-ee7b3c4d9049 node DatanodeRegistration(127.0.0.1:35079, datanodeUuid=c979ebad-dd51-47f8-91dc-d332f576cf02, infoPort=46661, infoSecurePort=0, ipcPort=41389, storageInfo=lv=-57;cid=testClusterID;nsid=1758385458;c=1731513936542), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:05:37,230 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661
2024-11-13T16:05:37,233 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/zookeeper_0, clientPort=55152, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:05:37,234 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55152
2024-11-13T16:05:37,234 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,235 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,243 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:05:37,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:05:37,245 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d with version=8
2024-11-13T16:05:37,245 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:05:37,247 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:05:37,248 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39213
2024-11-13T16:05:37,249 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39213 connecting to ZooKeeper ensemble=127.0.0.1:55152
2024-11-13T16:05:37,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:392130x0, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:05:37,255 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39213-0x100a6097d700000 connected
2024-11-13T16:05:37,271 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,273 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,274 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:37,275 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d, hbase.cluster.distributed=false
2024-11-13T16:05:37,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:05:37,276 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39213
2024-11-13T16:05:37,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39213
2024-11-13T16:05:37,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39213
2024-11-13T16:05:37,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39213
2024-11-13T16:05:37,277 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39213
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:05:37,293 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:05:37,294 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40409
2024-11-13T16:05:37,295 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40409 connecting to ZooKeeper ensemble=127.0.0.1:55152
2024-11-13T16:05:37,296 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,297 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404090x0, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:05:37,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40409-0x100a6097d700001 connected
2024-11-13T16:05:37,302 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:05:37,302 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:05:37,303 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:05:37,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:05:37,304 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:05:37,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40409
2024-11-13T16:05:37,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40409
2024-11-13T16:05:37,305 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40409
2024-11-13T16:05:37,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40409
2024-11-13T16:05:37,306 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40409
2024-11-13T16:05:37,317 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:39213
2024-11-13T16:05:37,318 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:37,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:37,320 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:05:37,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,323 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:05:37,323 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,39213,1731513937247 from backup master directory
2024-11-13T16:05:37,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:37,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:05:37,325 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:05:37,325 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,329 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/hbase.id] with ID: 74fc96ba-8206-40b2-9e09-04ca61911d12
2024-11-13T16:05:37,329 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/.tmp/hbase.id
2024-11-13T16:05:37,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:05:37,334 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:05:37,335 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/.tmp/hbase.id]:[hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/hbase.id]
2024-11-13T16:05:37,345 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:37,345 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:05:37,346 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-13T16:05:37,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:05:37,356 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:05:37,357 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:05:37,357 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:05:37,358 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:37,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:05:37,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:05:37,365 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store
2024-11-13T16:05:37,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:05:37,373 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:05:37,374 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:37,374 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:05:37,374 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513937374Disabling compacts and flushes for region at 1731513937374Disabling writes for close at 1731513937374Writing region close event to WAL at 1731513937374Closed at 1731513937374
2024-11-13T16:05:37,375 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/.initializing
2024-11-13T16:05:37,375 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/WALs/c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,377 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C39213%2C1731513937247, suffix=, logDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/WALs/c4c650e80f58,39213,1731513937247, archiveDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/oldWALs, maxLogs=10
2024-11-13T16:05:37,377 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C39213%2C1731513937247.1731513937377
2024-11-13T16:05:37,382 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/WALs/c4c650e80f58,39213,1731513937247/c4c650e80f58%2C39213%2C1731513937247.1731513937377
2024-11-13T16:05:37,387 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46661:46661),(127.0.0.1/127.0.0.1:39099:39099)]
2024-11-13T16:05:37,387 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:37,387 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:37,388 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,388 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,389 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:37,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:05:37,391 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:37,391 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:37,393 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:05:37,393 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,393 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:37,393 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:37,394 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:05:37,394 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,395 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:37,395 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:05:37,396 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:05:37,396 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,397 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:37,397 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,398 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,398 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,399 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,399 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,400 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:05:37,401 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:05:37,403 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:37,403 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773687, jitterRate=-0.016206786036491394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:05:37,404 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513937388Initializing all the Stores at 1731513937389 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513937389Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513937389Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513937389Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513937389Cleaning up temporary data from old regions at 1731513937399 (+10 ms)Region opened successfully at 1731513937404 (+5 ms)
2024-11-13T16:05:37,404 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:05:37,407 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f143a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:05:37,408 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:05:37,408 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:05:37,408 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:05:37,408 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:05:37,409 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:05:37,409 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:05:37,409 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:05:37,411 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:05:37,412 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:05:37,413 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:05:37,414 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:05:37,414 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:05:37,417 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:05:37,417 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:05:37,418 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:05:37,419 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:05:37,420 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:05:37,421 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:05:37,423 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:05:37,424 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:05:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:05:37,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,427 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,39213,1731513937247, sessionid=0x100a6097d700000, setting cluster-up flag (Was=false)
2024-11-13T16:05:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,437 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:05:37,438 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,448 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:05:37,449 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,39213,1731513937247
2024-11-13T16:05:37,451 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:05:37,452 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:37,453 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:05:37,453 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:05:37,453 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,39213,1731513937247
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:05:37,454 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:37,454 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:37,454 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:37,454 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:05:37,455 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:05:37,455 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,455 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:05:37,455 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,455 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731513967455
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:05:37,456 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:05:37,456 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:37,457 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:05:37,457 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:05:37,457 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:05:37,457 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:05:37,458 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,458 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:05:37,458 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513937457,5,FailOnTimeoutGroup]
2024-11-13T16:05:37,462 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513937458,5,FailOnTimeoutGroup]
2024-11-13T16:05:37,462 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,462 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:05:37,462 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,462 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:05:37,470 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:05:37,471 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:05:37,471 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d
2024-11-13T16:05:37,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:05:37,477 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:05:37,478 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:37,479 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:05:37,480 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:05:37,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,481 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:37,481 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:05:37,482 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:05:37,482 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,482 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:37,483 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:05:37,484 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:05:37,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:37,484 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:05:37,485 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:05:37,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:37,485 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:37,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:05:37,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740
2024-11-13T16:05:37,486 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740
2024-11-13T16:05:37,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:05:37,488 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:05:37,488 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:05:37,489 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:05:37,491 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:37,491 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=884722, jitterRate=0.12498302757740021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513937478Initializing all the Stores at 1731513937479 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513937479Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513937479Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513937479Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513937479Cleaning up temporary data from old regions at 1731513937488 (+9 ms)Region opened successfully at 1731513937492 (+4 ms)
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:05:37,492 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:05:37,492 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:05:37,492 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513937492Disabling compacts and flushes for region at 1731513937492Disabling writes for close at 1731513937492Writing region close event to WAL at 1731513937492Closed at 1731513937492
2024-11-13T16:05:37,494 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:37,494 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:05:37,494 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:05:37,495 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:05:37,497 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:05:37,507 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(746): ClusterId : 74fc96ba-8206-40b2-9e09-04ca61911d12
2024-11-13T16:05:37,508 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:05:37,510 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:05:37,510 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:05:37,513 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:05:37,514 DEBUG [RS:0;c4c650e80f58:40409 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cff114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:05:37,526 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:40409
2024-11-13T16:05:37,526 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:05:37,526 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:05:37,526 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:05:37,527 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,39213,1731513937247 with port=40409, startcode=1731513937293
2024-11-13T16:05:37,527 DEBUG [RS:0;c4c650e80f58:40409 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:05:37,529 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43871, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:05:37,530 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39213 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,530 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39213 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:37,532 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d
2024-11-13T16:05:37,532 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43247
2024-11-13T16:05:37,532 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:05:37,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:05:37,534 DEBUG [RS:0;c4c650e80f58:40409 {}] zookeeper.ZKUtil(111): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,534 WARN  [RS:0;c4c650e80f58:40409 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:05:37,534 INFO  [RS:0;c4c650e80f58:40409 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:37,534 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,535 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,40409,1731513937293]
2024-11-13T16:05:37,538 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:05:37,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:37,539 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:05:37,540 INFO  [RS:0;c4c650e80f58:40409 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:05:37,540 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,542 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:05:37,543 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:05:37,543 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,543 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,543 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,543 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,543 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,543 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:05:37,544 DEBUG [RS:0;c4c650e80f58:40409 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,547 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40409,1731513937293-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:05:37,564 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:05:37,564 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,40409,1731513937293-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,564 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,564 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.Replication(171): c4c650e80f58,40409,1731513937293 started
2024-11-13T16:05:37,579 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:37,580 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,40409,1731513937293, RpcServer on c4c650e80f58/172.17.0.3:40409, sessionid=0x100a6097d700001
2024-11-13T16:05:37,580 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:05:37,580 DEBUG [RS:0;c4c650e80f58:40409 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,580 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,40409,1731513937293'
2024-11-13T16:05:37,580 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,40409,1731513937293'
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:05:37,581 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:05:37,582 DEBUG [RS:0;c4c650e80f58:40409 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:05:37,582 INFO  [RS:0;c4c650e80f58:40409 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:05:37,582 INFO  [RS:0;c4c650e80f58:40409 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:05:37,647 WARN  [c4c650e80f58:39213 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:05:37,684 INFO  [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C40409%2C1731513937293, suffix=, logDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293, archiveDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs, maxLogs=32
2024-11-13T16:05:37,685 INFO  [RS:0;c4c650e80f58:40409 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40409%2C1731513937293.1731513937685
2024-11-13T16:05:37,691 INFO  [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513937685
2024-11-13T16:05:37,692 DEBUG [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39099:39099),(127.0.0.1/127.0.0.1:46661:46661)]
2024-11-13T16:05:37,897 DEBUG [c4c650e80f58:39213 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:05:37,898 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:37,899 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,40409,1731513937293, state=OPENING
2024-11-13T16:05:37,901 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:05:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:05:37,903 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:05:37,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:37,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:37,904 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,40409,1731513937293}]
2024-11-13T16:05:38,057 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:05:38,059 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56185, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:05:38,063 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:05:38,063 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:05:38,065 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C40409%2C1731513937293.meta, suffix=.meta, logDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293, archiveDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs, maxLogs=32
2024-11-13T16:05:38,065 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40409%2C1731513937293.meta.1731513938065.meta
2024-11-13T16:05:38,070 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.meta.1731513938065.meta
2024-11-13T16:05:38,072 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46661:46661),(127.0.0.1/127.0.0.1:39099:39099)]
2024-11-13T16:05:38,073 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:38,073 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:05:38,074 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:05:38,074 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:05:38,074 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:05:38,074 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:38,074 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:05:38,074 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:05:38,075 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:05:38,076 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:05:38,076 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:38,077 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:05:38,077 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:05:38,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:38,078 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:05:38,078 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:05:38,078 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,079 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:38,079 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:05:38,080 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:05:38,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,081 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:05:38,081 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:05:38,082 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740
2024-11-13T16:05:38,083 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740
2024-11-13T16:05:38,084 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:05:38,084 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:05:38,085 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:05:38,086 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:05:38,087 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=872709, jitterRate=0.10970774292945862}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:05:38,087 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:05:38,087 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513938074Writing region info on filesystem at 1731513938074Initializing all the Stores at 1731513938075 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513938075Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513938075Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513938075Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513938075Cleaning up temporary data from old regions at 1731513938084 (+9 ms)Running coprocessor post-open hooks at 1731513938087 (+3 ms)Region opened successfully at 1731513938087
2024-11-13T16:05:38,088 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513938057
2024-11-13T16:05:38,091 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:05:38,091 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:05:38,092 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:38,093 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,40409,1731513937293, state=OPEN
2024-11-13T16:05:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:05:38,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:05:38,099 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:38,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:38,099 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:05:38,102 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:05:38,102 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,40409,1731513937293 in 195 msec
2024-11-13T16:05:38,104 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:05:38,104 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec
2024-11-13T16:05:38,105 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:05:38,105 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:05:38,106 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:05:38,106 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,40409,1731513937293, seqNum=-1]
2024-11-13T16:05:38,107 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:05:38,108 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40219, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:05:38,114 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 661 msec
2024-11-13T16:05:38,114 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513938114, completionTime=-1
2024-11-13T16:05:38,114 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:05:38,114 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731513998116
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514058116
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:39213, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,116 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,118 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.795sec
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:05:38,120 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:05:38,123 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:05:38,123 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:05:38,123 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,39213,1731513937247-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:05:38,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0d70d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:38,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,39213,-1 for getting cluster id
2024-11-13T16:05:38,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:05:38,210 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '74fc96ba-8206-40b2-9e09-04ca61911d12'
2024-11-13T16:05:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:05:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "74fc96ba-8206-40b2-9e09-04ca61911d12"

2024-11-13T16:05:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2405a0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,39213,-1]
2024-11-13T16:05:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:05:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:05:38,213 INFO  [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:05:38,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4acc1c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:05:38,214 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:05:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,40409,1731513937293, seqNum=-1]
2024-11-13T16:05:38,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:05:38,216 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:05:38,218 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,39213,1731513937247
2024-11-13T16:05:38,218 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:05:38,221 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:05:38,222 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-13T16:05:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is c4c650e80f58,39213,1731513937247
2024-11-13T16:05:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b6f617
2024-11-13T16:05:38,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-13T16:05:38,224 INFO  [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-13T16:05:38,225 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-13T16:05:38,225 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-13T16:05:38,225 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:05:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:38,228 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-13T16:05:38,228 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,228 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling"
 procId is: 4
2024-11-13T16:05:38,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:05:38,229 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-13T16:05:38,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741835_1011 (size=405)
2024-11-13T16:05:38,236 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741835_1011 (size=405)
2024-11-13T16:05:38,238 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5773bd6af7f000b2f2753477ddcda49, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d
2024-11-13T16:05:38,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741836_1012 (size=88)
2024-11-13T16:05:38,244 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741836_1012 (size=88)
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing a5773bd6af7f000b2f2753477ddcda49, disabling compactions & flushes
2024-11-13T16:05:38,245 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49. after waiting 0 ms
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,245 INFO  [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,245 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5773bd6af7f000b2f2753477ddcda49:
Waiting for close lock at 1731513938245Disabling compacts and flushes for region at 1731513938245Disabling writes for close at 1731513938245Writing region close event to WAL at 1731513938245Closed at 1731513938245
2024-11-13T16:05:38,247 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-13T16:05:38,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731513938247"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731513938247"}]},"ts":"1731513938247"}
2024-11-13T16:05:38,250 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-13T16:05:38,251 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-13T16:05:38,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513938251"}]},"ts":"1731513938251"}
2024-11-13T16:05:38,253 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta
2024-11-13T16:05:38,253 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=a5773bd6af7f000b2f2753477ddcda49, ASSIGN}]
2024-11-13T16:05:38,255 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=a5773bd6af7f000b2f2753477ddcda49, ASSIGN
2024-11-13T16:05:38,256 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=a5773bd6af7f000b2f2753477ddcda49, ASSIGN; state=OFFLINE, location=c4c650e80f58,40409,1731513937293; forceNewPlan=false, retain=false
2024-11-13T16:05:38,406 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5773bd6af7f000b2f2753477ddcda49, regionState=OPENING, regionLocation=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:38,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=a5773bd6af7f000b2f2753477ddcda49, ASSIGN because future has completed
2024-11-13T16:05:38,410 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5773bd6af7f000b2f2753477ddcda49, server=c4c650e80f58,40409,1731513937293}]
2024-11-13T16:05:38,531 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:38,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:38,567 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,567 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a5773bd6af7f000b2f2753477ddcda49, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:05:38,567 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,567 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:05:38,567 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,567 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,569 INFO  [StoreOpener-a5773bd6af7f000b2f2753477ddcda49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region a5773bd6af7f000b2f2753477ddcda49 
2024-11-13T16:05:38,570 INFO  [StoreOpener-a5773bd6af7f000b2f2753477ddcda49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5773bd6af7f000b2f2753477ddcda49 columnFamilyName info
2024-11-13T16:05:38,570 DEBUG [StoreOpener-a5773bd6af7f000b2f2753477ddcda49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:05:38,571 INFO  [StoreOpener-a5773bd6af7f000b2f2753477ddcda49-1 {}] regionserver.HStore(327): Store=a5773bd6af7f000b2f2753477ddcda49/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:05:38,571 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,571 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,572 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,572 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,572 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,573 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,575 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:05:38,576 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a5773bd6af7f000b2f2753477ddcda49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836793, jitterRate=0.06403833627700806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:05:38,576 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:05:38,576 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a5773bd6af7f000b2f2753477ddcda49:
Running coprocessor pre-open hook at 1731513938567Writing region info on filesystem at 1731513938567Initializing all the Stores at 1731513938568 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513938568Cleaning up temporary data from old regions at 1731513938572 (+4 ms)Running coprocessor post-open hooks at 1731513938576 (+4 ms)Region opened successfully at 1731513938576
2024-11-13T16:05:38,577 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49., pid=6, masterSystemTime=1731513938563
2024-11-13T16:05:38,580 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,580 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:38,581 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a5773bd6af7f000b2f2753477ddcda49, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,40409,1731513937293
2024-11-13T16:05:38,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5773bd6af7f000b2f2753477ddcda49, server=c4c650e80f58,40409,1731513937293 because future has completed
2024-11-13T16:05:38,587 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-13T16:05:38,587 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a5773bd6af7f000b2f2753477ddcda49, server=c4c650e80f58,40409,1731513937293 in 174 msec
2024-11-13T16:05:38,589 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-13T16:05:38,589 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=a5773bd6af7f000b2f2753477ddcda49, ASSIGN in 334 msec
2024-11-13T16:05:38,590 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-13T16:05:38,591 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513938590"}]},"ts":"1731513938590"}
2024-11-13T16:05:38,593 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta
2024-11-13T16:05:38,594 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-13T16:05:38,595 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 369 msec
2024-11-13T16:05:39,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:39,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:40,532 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:40,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:41,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:41,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:41,872 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:05:41,873 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,892 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,895 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,895 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,896 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:41,898 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:05:42,533 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:42,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:43,534 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:43,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-13T16:05:43,538 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling'
2024-11-13T16:05:43,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:44,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:44,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:45,535 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:45,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:46,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta
2024-11-13T16:05:46,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer
2024-11-13T16:05:46,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:05:46,371 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers
2024-11-13T16:05:46,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store
2024-11-13T16:05:46,371 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer
2024-11-13T16:05:46,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:46,372 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer
2024-11-13T16:05:46,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:46,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:47,536 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:47,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:05:48,264 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-13T16:05:48,264 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100
2024-11-13T16:05:48,267 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:48,267 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49., hostname=c4c650e80f58,40409,1731513937293, seqNum=2]
2024-11-13T16:05:48,277 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:48,284 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-13T16:05:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-13T16:05:48,285 INFO  [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-13T16:05:48,287 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-13T16:05:48,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40409 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8
2024-11-13T16:05:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:48,448 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing a5773bd6af7f000b2f2753477ddcda49 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-13T16:05:48,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fe8f1b295bba4d7db5938bad2ff57b2f is 1080, key is row0001/info:/1731513948271/Put/seqid=0
2024-11-13T16:05:48,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741837_1013 (size=6033)
2024-11-13T16:05:48,472 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741837_1013 (size=6033)
2024-11-13T16:05:48,537 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:48,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:48,873 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fe8f1b295bba4d7db5938bad2ff57b2f
2024-11-13T16:05:48,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fe8f1b295bba4d7db5938bad2ff57b2f as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f
2024-11-13T16:05:48,886 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f, entries=1, sequenceid=5, filesize=5.9 K
2024-11-13T16:05:48,887 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 439ms, sequenceid=5, compaction requested=false
2024-11-13T16:05:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for a5773bd6af7f000b2f2753477ddcda49:

2024-11-13T16:05:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8
2024-11-13T16:05:48,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster(4169): Remote procedure done, pid=8
2024-11-13T16:05:48,895 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-13T16:05:48,895 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 605 msec
2024-11-13T16:05:48,898 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 617 msec
2024-11-13T16:05:49,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:49,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:50,538 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:50,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:51,539 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:51,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:52,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:52,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:53,540 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:53,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:54,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:54,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:55,541 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:55,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:56,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:56,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:57,542 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:57,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7
2024-11-13T16:05:58,335 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-13T16:05:58,338 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:05:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-13T16:05:58,340 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-13T16:05:58,341 INFO  [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-13T16:05:58,341 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-13T16:05:58,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40409 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10
2024-11-13T16:05:58,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:58,495 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing a5773bd6af7f000b2f2753477ddcda49 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-13T16:05:58,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/7bc90a9a462e43bea359b0bd55797c36 is 1080, key is row0002/info:/1731513958336/Put/seqid=0
2024-11-13T16:05:58,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741838_1014 (size=6033)
2024-11-13T16:05:58,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741838_1014 (size=6033)
2024-11-13T16:05:58,505 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/7bc90a9a462e43bea359b0bd55797c36
2024-11-13T16:05:58,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/7bc90a9a462e43bea359b0bd55797c36 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36
2024-11-13T16:05:58,517 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36, entries=1, sequenceid=9, filesize=5.9 K
2024-11-13T16:05:58,518 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 23ms, sequenceid=9, compaction requested=false
2024-11-13T16:05:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for a5773bd6af7f000b2f2753477ddcda49:

2024-11-13T16:05:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:05:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10
2024-11-13T16:05:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster(4169): Remote procedure done, pid=10
2024-11-13T16:05:58,522 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9
2024-11-13T16:05:58,522 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec
2024-11-13T16:05:58,525 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec
2024-11-13T16:05:58,543 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:58,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:59,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:05:59,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:00,544 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:00,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:01,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:01,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:02,545 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:02,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337 after 68048ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:06:02,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:02,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta after 68036ms
java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at jdk.internal.reflect.GeneratedMethodAccessor195.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
2024-11-13T16:06:03,546 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:03,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:04,547 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:04,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:05,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:05,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:06,548 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:06,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:07,230 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:06:07,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:07,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9
2024-11-13T16:06:08,435 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-13T16:06:08,438 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40409%2C1731513937293.1731513968438
2024-11-13T16:06:08,444 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:08,445 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:08,445 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:08,445 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:08,445 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:08,445 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513937685 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513968438
2024-11-13T16:06:08,446 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46661:46661),(127.0.0.1/127.0.0.1:39099:39099)]
2024-11-13T16:06:08,446 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513937685 is not closed yet, will try archiving it next time
2024-11-13T16:06:08,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741833_1009 (size=5546)
2024-11-13T16:06:08,447 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:06:08,447 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741833_1009 (size=5546)
2024-11-13T16:06:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:06:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-13T16:06:08,450 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-13T16:06:08,451 INFO  [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-13T16:06:08,451 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-13T16:06:08,549 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:08,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:08,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40409 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12
2024-11-13T16:06:08,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:08,605 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing a5773bd6af7f000b2f2753477ddcda49 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-13T16:06:08,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/3db17093fd074ce28bc7fa85859c35a6 is 1080, key is row0003/info:/1731513968436/Put/seqid=0
2024-11-13T16:06:08,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741840_1016 (size=6033)
2024-11-13T16:06:08,614 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741840_1016 (size=6033)
2024-11-13T16:06:08,614 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/3db17093fd074ce28bc7fa85859c35a6
2024-11-13T16:06:08,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/3db17093fd074ce28bc7fa85859c35a6 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6
2024-11-13T16:06:08,626 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6, entries=1, sequenceid=13, filesize=5.9 K
2024-11-13T16:06:08,627 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 22ms, sequenceid=13, compaction requested=true
2024-11-13T16:06:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for a5773bd6af7f000b2f2753477ddcda49:

2024-11-13T16:06:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:08,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12
2024-11-13T16:06:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster(4169): Remote procedure done, pid=12
2024-11-13T16:06:08,631 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-13T16:06:08,632 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 178 msec
2024-11-13T16:06:08,635 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec
2024-11-13T16:06:09,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:09,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:10,550 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:10,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:11,551 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:11,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:12,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:12,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:13,552 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:13,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:14,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:14,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:15,553 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:15,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:16,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:16,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:17,554 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:17,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:18,124 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-13T16:06:18,124 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-13T16:06:18,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11
2024-11-13T16:06:18,524 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-13T16:06:18,524 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:06:18,525 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:06:18,525 DEBUG [Time-limited test {}] regionserver.HStore(1541): a5773bd6af7f000b2f2753477ddcda49/info is initiating minor compaction (all files)
2024-11-13T16:06:18,526 INFO  [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:06:18,526 INFO  [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:18,526 INFO  [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a5773bd6af7f000b2f2753477ddcda49/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:18,526 INFO  [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6] into tmpdir=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp, totalSize=17.7 K
2024-11-13T16:06:18,526 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fe8f1b295bba4d7db5938bad2ff57b2f, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731513948271
2024-11-13T16:06:18,527 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7bc90a9a462e43bea359b0bd55797c36, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731513958336
2024-11-13T16:06:18,527 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3db17093fd074ce28bc7fa85859c35a6, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731513968436
2024-11-13T16:06:18,537 INFO  [Time-limited test {}] throttle.PressureAwareThroughputController(145): a5773bd6af7f000b2f2753477ddcda49#info#compaction#47 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:06:18,537 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/6b3b6c05c9e34d11816f5b940aa48228 is 1080, key is row0001/info:/1731513948271/Put/seqid=0
2024-11-13T16:06:18,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741841_1017 (size=8296)
2024-11-13T16:06:18,542 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741841_1017 (size=8296)
2024-11-13T16:06:18,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/6b3b6c05c9e34d11816f5b940aa48228 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/6b3b6c05c9e34d11816f5b940aa48228
2024-11-13T16:06:18,553 INFO  [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5773bd6af7f000b2f2753477ddcda49/info of a5773bd6af7f000b2f2753477ddcda49 into 6b3b6c05c9e34d11816f5b940aa48228(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:06:18,553 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a5773bd6af7f000b2f2753477ddcda49:

2024-11-13T16:06:18,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:18,556 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40409%2C1731513937293.1731513978556
2024-11-13T16:06:18,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:18,563 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:18,563 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:18,563 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:18,563 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:18,564 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:18,564 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513968438 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513978556
2024-11-13T16:06:18,564 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46661:46661),(127.0.0.1/127.0.0.1:39099:39099)]
2024-11-13T16:06:18,565 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513968438 is not closed yet, will try archiving it next time
2024-11-13T16:06:18,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741839_1015 (size=2520)
2024-11-13T16:06:18,565 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513937685 to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs/c4c650e80f58%2C40409%2C1731513937293.1731513937685
2024-11-13T16:06:18,565 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741839_1015 (size=2520)
2024-11-13T16:06:18,566 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:06:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:06:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-13T16:06:18,568 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE
2024-11-13T16:06:18,569 INFO  [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS
2024-11-13T16:06:18,569 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}]
2024-11-13T16:06:18,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40409 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14
2024-11-13T16:06:18,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:18,723 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing a5773bd6af7f000b2f2753477ddcda49 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-13T16:06:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/71c967376d6f4e0fb00f2ce74d5b32ef is 1080, key is row0000/info:/1731513978554/Put/seqid=0
2024-11-13T16:06:18,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741843_1019 (size=6033)
2024-11-13T16:06:18,732 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741843_1019 (size=6033)
2024-11-13T16:06:18,733 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/71c967376d6f4e0fb00f2ce74d5b32ef
2024-11-13T16:06:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/71c967376d6f4e0fb00f2ce74d5b32ef as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/71c967376d6f4e0fb00f2ce74d5b32ef
2024-11-13T16:06:18,743 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/71c967376d6f4e0fb00f2ce74d5b32ef, entries=1, sequenceid=18, filesize=5.9 K
2024-11-13T16:06:18,744 INFO  [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 21ms, sequenceid=18, compaction requested=false
2024-11-13T16:06:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for a5773bd6af7f000b2f2753477ddcda49:

2024-11-13T16:06:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:18,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14
2024-11-13T16:06:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.HMaster(4169): Remote procedure done, pid=14
2024-11-13T16:06:18,749 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13
2024-11-13T16:06:18,749 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 177 msec
2024-11-13T16:06:18,751 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 184 msec
2024-11-13T16:06:19,555 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:19,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:20,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:20,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:21,556 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:21,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:22,557 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:22,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:23,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:23,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:23,567 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region a5773bd6af7f000b2f2753477ddcda49, had cached 0 bytes from a total of 14329
2024-11-13T16:06:24,558 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:24,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:25,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:25,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:26,559 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:26,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:27,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:27,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:28,560 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:28,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39213 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13
2024-11-13T16:06:28,595 INFO  [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed
2024-11-13T16:06:28,598 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C40409%2C1731513937293.1731513988597
2024-11-13T16:06:28,603 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,604 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,604 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,604 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,604 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,604 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513978556 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513988597
2024-11-13T16:06:28,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741842_1018 (size=2026)
2024-11-13T16:06:28,606 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741842_1018 (size=2026)
2024-11-13T16:06:28,606 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/WALs/c4c650e80f58,40409,1731513937293/c4c650e80f58%2C40409%2C1731513937293.1731513968438 to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs/c4c650e80f58%2C40409%2C1731513937293.1731513968438
2024-11-13T16:06:28,607 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39099:39099),(127.0.0.1/127.0.0.1:46661:46661)]
2024-11-13T16:06:28,607 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:06:28,607 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:06:28,607 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:06:28,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:28,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:28,607 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:06:28,607 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:06:28,608 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=214782153, stopped=false
2024-11-13T16:06:28,608 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,39213,1731513937247
2024-11-13T16:06:28,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:06:28,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:06:28,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:28,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:28,610 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:06:28,610 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:06:28,610 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:06:28,610 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:28,610 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:06:28,610 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,40409,1731513937293' *****
2024-11-13T16:06:28,610 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:06:28,610 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:06:28,611 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(3091): Received CLOSE for a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,40409,1731513937293
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:06:28,611 INFO  [RS:0;c4c650e80f58:40409 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:40409.
2024-11-13T16:06:28,612 DEBUG [RS:0;c4c650e80f58:40409 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:06:28,612 DEBUG [RS:0;c4c650e80f58:40409 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5773bd6af7f000b2f2753477ddcda49, disabling compactions & flushes
2024-11-13T16:06:28,612 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:06:28,612 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:28,612 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:06:28,612 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:06:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:28,612 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:06:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49. after waiting 0 ms
2024-11-13T16:06:28,612 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:28,612 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a5773bd6af7f000b2f2753477ddcda49 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB
2024-11-13T16:06:28,613 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close
2024-11-13T16:06:28,613 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, a5773bd6af7f000b2f2753477ddcda49=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.}
2024-11-13T16:06:28,613 DEBUG [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, a5773bd6af7f000b2f2753477ddcda49
2024-11-13T16:06:28,613 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:06:28,613 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:06:28,613 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:06:28,613 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:06:28,613 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:06:28,613 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB
2024-11-13T16:06:28,616 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fca89621425f4778ae45e6fde087a086 is 1080, key is row0001/info:/1731513988596/Put/seqid=0
2024-11-13T16:06:28,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741845_1021 (size=6033)
2024-11-13T16:06:28,621 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741845_1021 (size=6033)
2024-11-13T16:06:28,622 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fca89621425f4778ae45e6fde087a086
2024-11-13T16:06:28,629 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/.tmp/info/fca89621425f4778ae45e6fde087a086 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fca89621425f4778ae45e6fde087a086
2024-11-13T16:06:28,632 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/info/2a764905da8b48e599be1437751e2ff0 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49./info:regioninfo/1731513938581/Put/seqid=0
2024-11-13T16:06:28,634 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fca89621425f4778ae45e6fde087a086, entries=1, sequenceid=22, filesize=5.9 K
2024-11-13T16:06:28,635 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 23ms, sequenceid=22, compaction requested=true
2024-11-13T16:06:28,636 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6] to archive
2024-11-13T16:06:28,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741846_1022 (size=7308)
2024-11-13T16:06:28,636 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741846_1022 (size=7308)
2024-11-13T16:06:28,637 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:06:28,637 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/info/2a764905da8b48e599be1437751e2ff0
2024-11-13T16:06:28,639 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/fe8f1b295bba4d7db5938bad2ff57b2f
2024-11-13T16:06:28,640 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36 to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/7bc90a9a462e43bea359b0bd55797c36
2024-11-13T16:06:28,642 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6 to hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/info/3db17093fd074ce28bc7fa85859c35a6
2024-11-13T16:06:28,642 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c4c650e80f58:39213 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-13T16:06:28,643 WARN  [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [fe8f1b295bba4d7db5938bad2ff57b2f=6033, 7bc90a9a462e43bea359b0bd55797c36=6033, 3db17093fd074ce28bc7fa85859c35a6=6033]
2024-11-13T16:06:28,646 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/a5773bd6af7f000b2f2753477ddcda49/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1
2024-11-13T16:06:28,647 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:28,647 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5773bd6af7f000b2f2753477ddcda49:
Waiting for close lock at 1731513988612Running coprocessor pre-close hooks at 1731513988612Disabling compacts and flushes for region at 1731513988612Disabling writes for close at 1731513988612Obtaining lock to block concurrent updates at 1731513988612Preparing flush snapshotting stores in a5773bd6af7f000b2f2753477ddcda49 at 1731513988612Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731513988612Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49. at 1731513988613 (+1 ms)Flushing a5773bd6af7f000b2f2753477ddcda49/info: creating writer at 1731513988613Flushing a5773bd6af7f000b2f2753477ddcda49/info: appending metadata at 1731513988616 (+3 ms)Flushing a5773bd6af7f000b2f2753477ddcda49/info: closing flushed file at 1731513988616Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bc2790e: reopening flushed file at 1731513988628 (+12 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for a5773bd6af7f000b2f2753477ddcda49 in 23ms, sequenceid=22, compaction requested=true at 1731513988635 (+7 ms)Writing region close event to WAL at 1731513988643 (+8 ms)Running coprocessor post-close hooks at 1731513988647 (+4 ms)Closed at 1731513988647
2024-11-13T16:06:28,647 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731513938225.a5773bd6af7f000b2f2753477ddcda49.
2024-11-13T16:06:28,657 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/ns/99f02f9cb5b442c1b9a384dcbbbe73b9 is 43, key is default/ns:d/1731513938109/Put/seqid=0
2024-11-13T16:06:28,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741847_1023 (size=5153)
2024-11-13T16:06:28,662 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741847_1023 (size=5153)
2024-11-13T16:06:28,663 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/ns/99f02f9cb5b442c1b9a384dcbbbe73b9
2024-11-13T16:06:28,689 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/table/8ef402814018449582b25e980ab57d4c is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731513938590/Put/seqid=0
2024-11-13T16:06:28,693 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741848_1024 (size=5508)
2024-11-13T16:06:28,694 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741848_1024 (size=5508)
2024-11-13T16:06:28,694 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/table/8ef402814018449582b25e980ab57d4c
2024-11-13T16:06:28,700 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/info/2a764905da8b48e599be1437751e2ff0 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/info/2a764905da8b48e599be1437751e2ff0
2024-11-13T16:06:28,705 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/info/2a764905da8b48e599be1437751e2ff0, entries=10, sequenceid=11, filesize=7.1 K
2024-11-13T16:06:28,706 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/ns/99f02f9cb5b442c1b9a384dcbbbe73b9 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/ns/99f02f9cb5b442c1b9a384dcbbbe73b9
2024-11-13T16:06:28,711 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/ns/99f02f9cb5b442c1b9a384dcbbbe73b9, entries=2, sequenceid=11, filesize=5.0 K
2024-11-13T16:06:28,712 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/.tmp/table/8ef402814018449582b25e980ab57d4c as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/table/8ef402814018449582b25e980ab57d4c
2024-11-13T16:06:28,716 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/table/8ef402814018449582b25e980ab57d4c, entries=2, sequenceid=11, filesize=5.4 K
2024-11-13T16:06:28,718 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false
2024-11-13T16:06:28,722 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1
2024-11-13T16:06:28,723 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:06:28,723 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:06:28,723 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513988613Running coprocessor pre-close hooks at 1731513988613Disabling compacts and flushes for region at 1731513988613Disabling writes for close at 1731513988613Obtaining lock to block concurrent updates at 1731513988613Preparing flush snapshotting stores in 1588230740 at 1731513988613Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731513988614 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731513988615 (+1 ms)Flushing 1588230740/info: creating writer at 1731513988615Flushing 1588230740/info: appending metadata at 1731513988631 (+16 ms)Flushing 1588230740/info: closing flushed file at 1731513988631Flushing 1588230740/ns: creating writer at 1731513988643 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731513988657 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731513988657Flushing 1588230740/table: creating writer at 1731513988668 (+11 ms)Flushing 1588230740/table: appending metadata at 1731513988688 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731513988688Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47e4ee60: reopening flushed file at 1731513988699 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@213dc142: reopening flushed file at 1731513988705 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1da8d836: reopening flushed file at 1731513988711 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false at 1731513988718 (+7 ms)Writing region close event to WAL at 1731513988719 (+1 ms)Running coprocessor post-close hooks at 1731513988723 (+4 ms)Closed at 1731513988723
2024-11-13T16:06:28,723 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:06:28,813 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,40409,1731513937293; all regions closed.
2024-11-13T16:06:28,814 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,814 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,814 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,814 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,814 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741834_1010 (size=3306)
2024-11-13T16:06:28,816 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741834_1010 (size=3306)
2024-11-13T16:06:28,819 DEBUG [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs
2024-11-13T16:06:28,819 INFO  [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C40409%2C1731513937293.meta:.meta(num 1731513938065)
2024-11-13T16:06:28,819 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,819 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,819 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,819 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,819 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741844_1020 (size=1252)
2024-11-13T16:06:28,821 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741844_1020 (size=1252)
2024-11-13T16:06:28,824 DEBUG [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/oldWALs
2024-11-13T16:06:28,824 INFO  [RS:0;c4c650e80f58:40409 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C40409%2C1731513937293:(num 1731513988597)
2024-11-13T16:06:28,824 DEBUG [RS:0;c4c650e80f58:40409 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:28,824 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:06:28,824 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:06:28,824 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-13T16:06:28,824 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:06:28,824 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:06:28,825 INFO  [RS:0;c4c650e80f58:40409 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40409
2024-11-13T16:06:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:06:28,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,40409,1731513937293
2024-11-13T16:06:28,827 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:06:28,828 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,40409,1731513937293]
2024-11-13T16:06:28,830 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,40409,1731513937293 already deleted, retry=false
2024-11-13T16:06:28,830 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,40409,1731513937293 expired; onlineServers=0
2024-11-13T16:06:28,830 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,39213,1731513937247' *****
2024-11-13T16:06:28,830 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:06:28,831 DEBUG [M:0;c4c650e80f58:39213 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:06:28,831 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:06:28,831 DEBUG [M:0;c4c650e80f58:39213 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:06:28,831 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513937457 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513937457,5,FailOnTimeoutGroup]
2024-11-13T16:06:28,831 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513937458 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513937458,5,FailOnTimeoutGroup]
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:06:28,831 DEBUG [M:0;c4c650e80f58:39213 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:06:28,831 INFO  [M:0;c4c650e80f58:39213 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:06:28,832 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:06:28,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:06:28,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:28,832 DEBUG [M:0;c4c650e80f58:39213 {}] zookeeper.ZKUtil(347): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:06:28,833 WARN  [M:0;c4c650e80f58:39213 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:06:28,833 INFO  [M:0;c4c650e80f58:39213 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/.lastflushedseqids
2024-11-13T16:06:28,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741849_1025 (size=130)
2024-11-13T16:06:28,838 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741849_1025 (size=130)
2024-11-13T16:06:28,839 INFO  [M:0;c4c650e80f58:39213 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:06:28,839 INFO  [M:0;c4c650e80f58:39213 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:06:28,839 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:06:28,839 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:28,839 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:28,839 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:06:28,839 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:28,839 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.54 KB heapSize=54.91 KB
2024-11-13T16:06:28,855 DEBUG [M:0;c4c650e80f58:39213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/327dac1119e34bf683101bafb6e9ec38 is 82, key is hbase:meta,,1/info:regioninfo/1731513938092/Put/seqid=0
2024-11-13T16:06:28,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741850_1026 (size=5672)
2024-11-13T16:06:28,859 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741850_1026 (size=5672)
2024-11-13T16:06:28,860 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/327dac1119e34bf683101bafb6e9ec38
2024-11-13T16:06:28,880 DEBUG [M:0;c4c650e80f58:39213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21fd0b32ee644b2aaa2a903e8498aa9b is 797, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731513938595/Put/seqid=0
2024-11-13T16:06:28,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741851_1027 (size=7818)
2024-11-13T16:06:28,885 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741851_1027 (size=7818)
2024-11-13T16:06:28,886 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.94 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21fd0b32ee644b2aaa2a903e8498aa9b
2024-11-13T16:06:28,890 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21fd0b32ee644b2aaa2a903e8498aa9b
2024-11-13T16:06:28,904 DEBUG [M:0;c4c650e80f58:39213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/edce04c83b0548e39cdcea1bffcc5608 is 69, key is c4c650e80f58,40409,1731513937293/rs:state/1731513937530/Put/seqid=0
2024-11-13T16:06:28,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741852_1028 (size=5156)
2024-11-13T16:06:28,909 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741852_1028 (size=5156)
2024-11-13T16:06:28,910 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/edce04c83b0548e39cdcea1bffcc5608
2024-11-13T16:06:28,928 DEBUG [M:0;c4c650e80f58:39213 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6bd67463f26d43bb93b95d6dbcddd011 is 52, key is load_balancer_on/state:d/1731513938220/Put/seqid=0
2024-11-13T16:06:28,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:06:28,928 INFO  [RS:0;c4c650e80f58:40409 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:06:28,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40409-0x100a6097d700001, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:06:28,929 INFO  [RS:0;c4c650e80f58:40409 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,40409,1731513937293; zookeeper connection closed.
2024-11-13T16:06:28,929 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7aec30cb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7aec30cb
2024-11-13T16:06:28,929 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:06:28,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741853_1029 (size=5056)
2024-11-13T16:06:28,933 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741853_1029 (size=5056)
2024-11-13T16:06:28,934 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6bd67463f26d43bb93b95d6dbcddd011
2024-11-13T16:06:28,939 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/327dac1119e34bf683101bafb6e9ec38 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/327dac1119e34bf683101bafb6e9ec38
2024-11-13T16:06:28,943 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/327dac1119e34bf683101bafb6e9ec38, entries=8, sequenceid=121, filesize=5.5 K
2024-11-13T16:06:28,944 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/21fd0b32ee644b2aaa2a903e8498aa9b as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21fd0b32ee644b2aaa2a903e8498aa9b
2024-11-13T16:06:28,948 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 21fd0b32ee644b2aaa2a903e8498aa9b
2024-11-13T16:06:28,948 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/21fd0b32ee644b2aaa2a903e8498aa9b, entries=14, sequenceid=121, filesize=7.6 K
2024-11-13T16:06:28,948 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/edce04c83b0548e39cdcea1bffcc5608 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/edce04c83b0548e39cdcea1bffcc5608
2024-11-13T16:06:28,953 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/edce04c83b0548e39cdcea1bffcc5608, entries=1, sequenceid=121, filesize=5.0 K
2024-11-13T16:06:28,954 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6bd67463f26d43bb93b95d6dbcddd011 as hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6bd67463f26d43bb93b95d6dbcddd011
2024-11-13T16:06:28,957 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43247/user/jenkins/test-data/d3eca9ef-ea9a-6238-ae77-e7db4c8ed57d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6bd67463f26d43bb93b95d6dbcddd011, entries=1, sequenceid=121, filesize=4.9 K
2024-11-13T16:06:28,958 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.54 KB/44590, heapSize ~54.85 KB/56168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=121, compaction requested=false
2024-11-13T16:06:28,960 INFO  [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:28,960 DEBUG [M:0;c4c650e80f58:39213 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513988839Disabling compacts and flushes for region at 1731513988839Disabling writes for close at 1731513988839Obtaining lock to block concurrent updates at 1731513988839Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731513988839Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44590, getHeapSize=56168, getOffHeapSize=0, getCellsCount=140 at 1731513988840 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731513988840Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731513988840Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731513988854 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731513988854Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731513988864 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731513988880 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731513988880Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731513988890 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731513988904 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731513988904Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731513988914 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731513988928 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731513988928Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fba813a: reopening flushed file at 1731513988938 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@260f16d5: reopening flushed file at 1731513988943 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@543aa764: reopening flushed file at 1731513988948 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2492c65d: reopening flushed file at 1731513988953 (+5 ms)Finished flush of dataSize ~43.54 KB/44590, heapSize ~54.85 KB/56168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 119ms, sequenceid=121, compaction requested=false at 1731513988958 (+5 ms)Writing region close event to WAL at 1731513988960 (+2 ms)Closed at 1731513988960
2024-11-13T16:06:28,960 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,960 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,960 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,961 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,961 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:06:28,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46833 is added to blk_1073741830_1006 (size=52987)
2024-11-13T16:06:28,962 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35079 is added to blk_1073741830_1006 (size=52987)
2024-11-13T16:06:28,963 INFO  [M:0;c4c650e80f58:39213 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:06:28,963 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:06:28,963 INFO  [M:0;c4c650e80f58:39213 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39213
2024-11-13T16:06:28,964 INFO  [M:0;c4c650e80f58:39213 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:06:29,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:06:29,065 INFO  [M:0;c4c650e80f58:39213 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:06:29,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39213-0x100a6097d700000, quorum=127.0.0.1:55152, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:06:29,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5daea1f2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:06:29,068 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@448bc78{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:06:29,068 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:06:29,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30873421{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:06:29,068 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e5fe848{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,STOPPED}
2024-11-13T16:06:29,070 WARN  [BP-607491404-172.17.0.3-1731513936542 heartbeating to localhost/127.0.0.1:43247 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:06:29,070 WARN  [BP-607491404-172.17.0.3-1731513936542 heartbeating to localhost/127.0.0.1:43247 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-607491404-172.17.0.3-1731513936542 (Datanode Uuid c979ebad-dd51-47f8-91dc-d332f576cf02) service to localhost/127.0.0.1:43247
2024-11-13T16:06:29,070 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:06:29,070 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data3/current/BP-607491404-172.17.0.3-1731513936542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:06:29,070 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:06:29,071 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data4/current/BP-607491404-172.17.0.3-1731513936542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:06:29,071 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:06:29,073 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bc64a6a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:06:29,073 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49fb27f0{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:06:29,073 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:06:29,073 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4e873b68{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:06:29,073 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dd1f450{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,STOPPED}
2024-11-13T16:06:29,075 WARN  [BP-607491404-172.17.0.3-1731513936542 heartbeating to localhost/127.0.0.1:43247 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:06:29,075 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:06:29,075 WARN  [BP-607491404-172.17.0.3-1731513936542 heartbeating to localhost/127.0.0.1:43247 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-607491404-172.17.0.3-1731513936542 (Datanode Uuid 48ca5872-ccbd-4932-989e-0b81f04c498a) service to localhost/127.0.0.1:43247
2024-11-13T16:06:29,075 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:06:29,076 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data1/current/BP-607491404-172.17.0.3-1731513936542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:06:29,076 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/cluster_4e9395d2-f607-193b-6f64-7b0257b9fea9/data/data2/current/BP-607491404-172.17.0.3-1731513936542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:06:29,076 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:06:29,082 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ecaedd6{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:06:29,082 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16aeea80{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:06:29,082 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:06:29,083 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d6dee42{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:06:29,083 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@510b5e02{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir/,STOPPED}
2024-11-13T16:06:29,089 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:06:29,105 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:06:29,115 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 179)
Potentially hanging thread: nioEventLoopGroup-36-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-12-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-36-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43247
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43247 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43247
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:43247
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16
	java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method)
	java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129)
	java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879)
	app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-34-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-37-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43247
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-35-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43247 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-12-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43247 from jenkins.hfs.5
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:43247
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: regionserver/c4c650e80f58:0.leaseChecker
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82)
 - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=69 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=11367 (was 11464)
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=69, ProcessCount=11, AvailableMemoryMB=11366
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.log.dir so I do NOT create it in target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b5200649-de5a-5913-b985-fe441a4fa661/hadoop.tmp.dir so I do NOT create it in target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23, deleteOnExit=true
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/test.cache.data in system properties and HBase conf
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:06:29,123 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:06:29,124 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:06:29,124 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:06:29,137 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:06:29,209 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:06:29,213 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:06:29,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:06:29,214 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:06:29,214 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:06:29,214 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:06:29,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a67ff9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:06:29,215 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45bda0cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:06:29,328 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51585bde{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/java.io.tmpdir/jetty-localhost-35065-hadoop-hdfs-3_4_1-tests_jar-_-any-1543690456795903361/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:06:29,328 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15c8c411{HTTP/1.1, (http/1.1)}{localhost:35065}
2024-11-13T16:06:29,328 INFO  [Time-limited test {}] server.Server(415): Started @238214ms
2024-11-13T16:06:29,341 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:06:29,416 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:06:29,418 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:06:29,419 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:06:29,419 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:06:29,419 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:06:29,420 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59fd7f75{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:06:29,420 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417c1a7a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:06:29,532 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4063a419{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/java.io.tmpdir/jetty-localhost-43499-hadoop-hdfs-3_4_1-tests_jar-_-any-17846867139239201919/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:06:29,533 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c0c72e{HTTP/1.1, (http/1.1)}{localhost:43499}
2024-11-13T16:06:29,533 INFO  [Time-limited test {}] server.Server(415): Started @238419ms
2024-11-13T16:06:29,534 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:06:29,550 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:06:29,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:29,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:29,565 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:06:29,567 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:06:29,568 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:06:29,568 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:06:29,568 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:06:29,568 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5773e0ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:06:29,569 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41c54b7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:06:29,625 WARN  [Thread-1973 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data1/current/BP-1342233215-172.17.0.3-1731513989155/current, will proceed with Du for space computation calculation, 
2024-11-13T16:06:29,625 WARN  [Thread-1974 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data2/current/BP-1342233215-172.17.0.3-1731513989155/current, will proceed with Du for space computation calculation, 
2024-11-13T16:06:29,641 WARN  [Thread-1952 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:06:29,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf35e98b172ecd57 with lease ID 0xbefc6091b71b1620: Processing first storage report for DS-effbe8c0-f1ba-49cf-8346-e0d3656de2b3 from datanode DatanodeRegistration(127.0.0.1:38629, datanodeUuid=96b784ac-5f7b-49a0-981c-64df61c81141, infoPort=40329, infoSecurePort=0, ipcPort=43231, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155)
2024-11-13T16:06:29,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf35e98b172ecd57 with lease ID 0xbefc6091b71b1620: from storage DS-effbe8c0-f1ba-49cf-8346-e0d3656de2b3 node DatanodeRegistration(127.0.0.1:38629, datanodeUuid=96b784ac-5f7b-49a0-981c-64df61c81141, infoPort=40329, infoSecurePort=0, ipcPort=43231, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:06:29,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf35e98b172ecd57 with lease ID 0xbefc6091b71b1620: Processing first storage report for DS-f11e434c-7d24-4478-b51e-ba458082575c from datanode DatanodeRegistration(127.0.0.1:38629, datanodeUuid=96b784ac-5f7b-49a0-981c-64df61c81141, infoPort=40329, infoSecurePort=0, ipcPort=43231, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155)
2024-11-13T16:06:29,643 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf35e98b172ecd57 with lease ID 0xbefc6091b71b1620: from storage DS-f11e434c-7d24-4478-b51e-ba458082575c node DatanodeRegistration(127.0.0.1:38629, datanodeUuid=96b784ac-5f7b-49a0-981c-64df61c81141, infoPort=40329, infoSecurePort=0, ipcPort=43231, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:06:29,684 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4707bb9a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/java.io.tmpdir/jetty-localhost-39327-hadoop-hdfs-3_4_1-tests_jar-_-any-6562329657042395575/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:06:29,684 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@286b8c80{HTTP/1.1, (http/1.1)}{localhost:39327}
2024-11-13T16:06:29,684 INFO  [Time-limited test {}] server.Server(415): Started @238570ms
2024-11-13T16:06:29,685 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:06:29,778 WARN  [Thread-1999 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data3/current/BP-1342233215-172.17.0.3-1731513989155/current, will proceed with Du for space computation calculation, 
2024-11-13T16:06:29,778 WARN  [Thread-2000 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data4/current/BP-1342233215-172.17.0.3-1731513989155/current, will proceed with Du for space computation calculation, 
2024-11-13T16:06:29,796 WARN  [Thread-1988 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:06:29,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed99fcebcd24d278 with lease ID 0xbefc6091b71b1621: Processing first storage report for DS-9276fed1-3270-4043-9a6b-033e79235ceb from datanode DatanodeRegistration(127.0.0.1:36533, datanodeUuid=c7b1ab68-bac6-414b-ab8d-4bc06d47bdbb, infoPort=42183, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155)
2024-11-13T16:06:29,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed99fcebcd24d278 with lease ID 0xbefc6091b71b1621: from storage DS-9276fed1-3270-4043-9a6b-033e79235ceb node DatanodeRegistration(127.0.0.1:36533, datanodeUuid=c7b1ab68-bac6-414b-ab8d-4bc06d47bdbb, infoPort=42183, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:06:29,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xed99fcebcd24d278 with lease ID 0xbefc6091b71b1621: Processing first storage report for DS-b33075f9-b386-475d-844d-d7ec7f3d72a3 from datanode DatanodeRegistration(127.0.0.1:36533, datanodeUuid=c7b1ab68-bac6-414b-ab8d-4bc06d47bdbb, infoPort=42183, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155)
2024-11-13T16:06:29,798 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xed99fcebcd24d278 with lease ID 0xbefc6091b71b1621: from storage DS-b33075f9-b386-475d-844d-d7ec7f3d72a3 node DatanodeRegistration(127.0.0.1:36533, datanodeUuid=c7b1ab68-bac6-414b-ab8d-4bc06d47bdbb, infoPort=42183, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=88441790;c=1731513989155), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:06:29,807 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957
2024-11-13T16:06:29,809 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/zookeeper_0, clientPort=63114, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:06:29,810 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63114
2024-11-13T16:06:29,810 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,811 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,818 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:06:29,819 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:06:29,820 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b with version=8
2024-11-13T16:06:29,820 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:06:29,821 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:06:29,822 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41807
2024-11-13T16:06:29,823 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41807 connecting to ZooKeeper ensemble=127.0.0.1:63114
2024-11-13T16:06:29,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418070x0, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:06:29,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41807-0x100a60a4ad00000 connected
2024-11-13T16:06:29,844 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,846 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,847 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:06:29,848 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b, hbase.cluster.distributed=false
2024-11-13T16:06:29,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:06:29,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41807
2024-11-13T16:06:29,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41807
2024-11-13T16:06:29,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41807
2024-11-13T16:06:29,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41807
2024-11-13T16:06:29,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41807
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:06:29,865 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:06:29,866 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:06:29,866 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44287
2024-11-13T16:06:29,867 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44287 connecting to ZooKeeper ensemble=127.0.0.1:63114
2024-11-13T16:06:29,868 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,869 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442870x0, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:06:29,874 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442870x0, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:06:29,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44287-0x100a60a4ad00001 connected
2024-11-13T16:06:29,874 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:06:29,874 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:06:29,875 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:06:29,876 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:06:29,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44287
2024-11-13T16:06:29,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44287
2024-11-13T16:06:29,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44287
2024-11-13T16:06:29,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44287
2024-11-13T16:06:29,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44287
2024-11-13T16:06:29,891 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:41807
2024-11-13T16:06:29,891 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,41807,1731513989821
2024-11-13T16:06:29,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:06:29,893 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:06:29,893 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,41807,1731513989821
2024-11-13T16:06:29,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:06:29,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,896 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:06:29,896 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,41807,1731513989821 from backup master directory
2024-11-13T16:06:29,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,41807,1731513989821
2024-11-13T16:06:29,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:06:29,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:06:29,898 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:06:29,898 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,41807,1731513989821
2024-11-13T16:06:29,901 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/hbase.id] with ID: 9a30051b-ad12-4913-bd0b-fb0270f87a48
2024-11-13T16:06:29,901 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/.tmp/hbase.id
2024-11-13T16:06:29,906 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:06:29,907 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:06:29,907 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/.tmp/hbase.id]:[hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/hbase.id]
2024-11-13T16:06:29,917 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:29,917 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:06:29,918 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-13T16:06:29,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:06:29,928 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:06:29,929 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:06:29,929 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:06:29,930 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:06:29,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:06:29,936 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:06:29,937 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store
2024-11-13T16:06:29,943 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:06:29,944 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:06:29,944 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:29,944 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:06:29,944 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731513989944Disabling compacts and flushes for region at 1731513989944Disabling writes for close at 1731513989944Writing region close event to WAL at 1731513989944Closed at 1731513989944
2024-11-13T16:06:29,945 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/.initializing
2024-11-13T16:06:29,945 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/WALs/c4c650e80f58,41807,1731513989821
2024-11-13T16:06:29,947 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C41807%2C1731513989821, suffix=, logDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/WALs/c4c650e80f58,41807,1731513989821, archiveDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/oldWALs, maxLogs=10
2024-11-13T16:06:29,948 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C41807%2C1731513989821.1731513989948
2024-11-13T16:06:29,953 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/WALs/c4c650e80f58,41807,1731513989821/c4c650e80f58%2C41807%2C1731513989821.1731513989948
2024-11-13T16:06:29,953 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42183:42183),(127.0.0.1/127.0.0.1:40329:40329)]
2024-11-13T16:06:29,954 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:06:29,954 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:29,954 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,954 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,957 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:06:29,958 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:06:29,958 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:29,958 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:29,959 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:06:29,959 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:06:29,959 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:29,960 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:29,960 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:06:29,961 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:06:29,961 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:29,961 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:29,961 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:06:29,962 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:06:29,962 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:29,963 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:29,963 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,963 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,964 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,965 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,965 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,965 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:06:29,966 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:06:29,968 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:06:29,968 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841317, jitterRate=0.06979039311408997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:06:29,969 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731513989954Initializing all the Stores at 1731513989955 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513989955Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513989957 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513989957Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513989957Cleaning up temporary data from old regions at 1731513989965 (+8 ms)Region opened successfully at 1731513989969 (+4 ms)
2024-11-13T16:06:29,969 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:06:29,971 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@155e8476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:06:29,972 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:06:29,972 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:06:29,972 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:06:29,972 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:06:29,973 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:06:29,973 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:06:29,973 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:06:29,975 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:06:29,976 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:06:29,977 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:06:29,977 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:06:29,978 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:06:29,980 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:06:29,980 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:06:29,981 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:06:29,982 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:06:29,983 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:06:29,984 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:06:29,986 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:06:29,987 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:06:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:06:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:06:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,990 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,41807,1731513989821, sessionid=0x100a60a4ad00000, setting cluster-up flag (Was=false)
2024-11-13T16:06:29,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:29,999 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:06:30,000 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,41807,1731513989821
2024-11-13T16:06:30,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:30,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:30,031 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:06:30,033 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,41807,1731513989821
2024-11-13T16:06:30,034 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:06:30,036 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:06:30,036 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:06:30,036 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:06:30,037 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,41807,1731513989821
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:06:30,038 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514020040
2024-11-13T16:06:30,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:06:30,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:06:30,042 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513990041,5,FailOnTimeoutGroup]
2024-11-13T16:06:30,042 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513990042,5,FailOnTimeoutGroup]
2024-11-13T16:06:30,042 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,042 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:06:30,042 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,042 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,042 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:06:30,042 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:06:30,044 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,044 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:06:30,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:06:30,051 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:06:30,052 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:06:30,052 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b
2024-11-13T16:06:30,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:06:30,058 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:06:30,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:30,060 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:06:30,061 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:06:30,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,062 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,062 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:06:30,063 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:06:30,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,064 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,064 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:06:30,065 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:06:30,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,065 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,065 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:06:30,066 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:06:30,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,067 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:06:30,067 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740
2024-11-13T16:06:30,068 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740
2024-11-13T16:06:30,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:06:30,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:06:30,069 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:06:30,070 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:06:30,072 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:06:30,072 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=796234, jitterRate=0.012464940547943115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731513990059Initializing all the Stores at 1731513990060 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990060Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990060Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513990060Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990060Cleaning up temporary data from old regions at 1731513990069 (+9 ms)Region opened successfully at 1731513990073 (+4 ms)
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:06:30,073 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:06:30,073 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:06:30,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731513990073Disabling compacts and flushes for region at 1731513990073Disabling writes for close at 1731513990073Writing region close event to WAL at 1731513990073Closed at 1731513990073
2024-11-13T16:06:30,075 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:06:30,075 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:06:30,075 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:06:30,076 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:06:30,077 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:06:30,081 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(746): ClusterId : 9a30051b-ad12-4913-bd0b-fb0270f87a48
2024-11-13T16:06:30,081 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:06:30,083 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:06:30,083 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:06:30,087 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:06:30,087 DEBUG [RS:0;c4c650e80f58:44287 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@118e2926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:06:30,099 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:44287
2024-11-13T16:06:30,099 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:06:30,099 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:06:30,099 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:06:30,100 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,41807,1731513989821 with port=44287, startcode=1731513989865
2024-11-13T16:06:30,100 DEBUG [RS:0;c4c650e80f58:44287 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:06:30,102 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44527, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:06:30,103 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,103 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41807 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,104 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b
2024-11-13T16:06:30,105 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34901
2024-11-13T16:06:30,105 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:06:30,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:06:30,107 DEBUG [RS:0;c4c650e80f58:44287 {}] zookeeper.ZKUtil(111): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,107 WARN  [RS:0;c4c650e80f58:44287 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:06:30,107 INFO  [RS:0;c4c650e80f58:44287 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:06:30,107 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,107 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,44287,1731513989865]
2024-11-13T16:06:30,110 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:06:30,112 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:06:30,112 INFO  [RS:0;c4c650e80f58:44287 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:06:30,112 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,113 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:06:30,113 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:06:30,113 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:06:30,114 DEBUG [RS:0;c4c650e80f58:44287 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,114 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,44287,1731513989865-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:06:30,129 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:06:30,129 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,44287,1731513989865-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,129 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,129 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.Replication(171): c4c650e80f58,44287,1731513989865 started
2024-11-13T16:06:30,143 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,143 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,44287,1731513989865, RpcServer on c4c650e80f58/172.17.0.3:44287, sessionid=0x100a60a4ad00001
2024-11-13T16:06:30,143 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:06:30,143 DEBUG [RS:0;c4c650e80f58:44287 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,143 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,44287,1731513989865'
2024-11-13T16:06:30,143 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:06:30,144 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:06:30,144 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:06:30,144 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:06:30,145 DEBUG [RS:0;c4c650e80f58:44287 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,145 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,44287,1731513989865'
2024-11-13T16:06:30,145 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:06:30,145 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:06:30,145 DEBUG [RS:0;c4c650e80f58:44287 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:06:30,145 INFO  [RS:0;c4c650e80f58:44287 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:06:30,145 INFO  [RS:0;c4c650e80f58:44287 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:06:30,227 WARN  [c4c650e80f58:41807 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:06:30,247 INFO  [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C44287%2C1731513989865, suffix=, logDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865, archiveDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs, maxLogs=32
2024-11-13T16:06:30,248 INFO  [RS:0;c4c650e80f58:44287 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44287%2C1731513989865.1731513990248
2024-11-13T16:06:30,253 INFO  [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731513990248
2024-11-13T16:06:30,254 DEBUG [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42183:42183),(127.0.0.1/127.0.0.1:40329:40329)]
2024-11-13T16:06:30,478 DEBUG [c4c650e80f58:41807 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:06:30,478 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,480 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,44287,1731513989865, state=OPENING
2024-11-13T16:06:30,481 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:06:30,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:30,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:06:30,483 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:06:30,483 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:06:30,483 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,44287,1731513989865}]
2024-11-13T16:06:30,483 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:06:30,561 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:30,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:30,636 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:06:30,638 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33503, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:06:30,641 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:06:30,641 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:06:30,643 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C44287%2C1731513989865.meta, suffix=.meta, logDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865, archiveDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs, maxLogs=32
2024-11-13T16:06:30,643 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44287%2C1731513989865.meta.1731513990643.meta
2024-11-13T16:06:30,648 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.meta.1731513990643.meta
2024-11-13T16:06:30,649 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40329:40329),(127.0.0.1/127.0.0.1:42183:42183)]
2024-11-13T16:06:30,649 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:06:30,650 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:06:30,650 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:06:30,651 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:06:30,652 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:06:30,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,652 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,652 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:06:30,653 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:06:30,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,653 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:06:30,654 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:06:30,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,654 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,654 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:06:30,655 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:06:30,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,655 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:06:30,655 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:06:30,656 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740
2024-11-13T16:06:30,657 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740
2024-11-13T16:06:30,658 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:06:30,658 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:06:30,658 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:06:30,659 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:06:30,660 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=801762, jitterRate=0.019493937492370605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:06:30,660 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:06:30,661 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731513990650Writing region info on filesystem at 1731513990650Initializing all the Stores at 1731513990651 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990651Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990651Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513990651Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731513990651Cleaning up temporary data from old regions at 1731513990658 (+7 ms)Running coprocessor post-open hooks at 1731513990660 (+2 ms)Region opened successfully at 1731513990661 (+1 ms)
2024-11-13T16:06:30,662 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731513990636
2024-11-13T16:06:30,664 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:06:30,664 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:06:30,665 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,665 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,44287,1731513989865, state=OPEN
2024-11-13T16:06:30,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:06:30,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:06:30,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:06:30,672 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,672 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:06:30,674 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:06:30,674 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,44287,1731513989865 in 189 msec
2024-11-13T16:06:30,676 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:06:30,676 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec
2024-11-13T16:06:30,677 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:06:30,677 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:06:30,678 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:06:30,678 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,44287,1731513989865, seqNum=-1]
2024-11-13T16:06:30,678 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:06:30,680 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:06:30,685 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 649 msec
2024-11-13T16:06:30,685 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731513990685, completionTime=-1
2024-11-13T16:06:30,685 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:06:30,685 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514050687
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514110687
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,687 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,688 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,688 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:41807, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,688 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,688 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,689 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.793sec
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:06:30,691 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:06:30,693 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:06:30,693 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:06:30,693 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,41807,1731513989821-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:06:30,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2983d230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:06:30,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,41807,-1 for getting cluster id
2024-11-13T16:06:30,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:06:30,783 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a30051b-ad12-4913-bd0b-fb0270f87a48'
2024-11-13T16:06:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:06:30,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a30051b-ad12-4913-bd0b-fb0270f87a48"

2024-11-13T16:06:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e9fb21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:06:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,41807,-1]
2024-11-13T16:06:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:06:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:06:30,785 INFO  [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:06:30,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d663ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:06:30,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:06:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,44287,1731513989865, seqNum=-1]
2024-11-13T16:06:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:06:30,788 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:06:30,789 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,41807,1731513989821
2024-11-13T16:06:30,790 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:06:30,792 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:06:30,792 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry
2024-11-13T16:06:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c4c650e80f58,41807,1731513989821
2024-11-13T16:06:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@34648aa
2024-11-13T16:06:30,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false
2024-11-13T16:06:30,794 INFO  [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService
2024-11-13T16:06:30,794 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions.
2024-11-13T16:06:30,794 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing.
2024-11-13T16:06:30,795 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:06:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling
2024-11-13T16:06:30,797 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION
2024-11-13T16:06:30,797 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:30,797 INFO  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default"
qualifier: "TestLogRolling-testLogRolling"
 procId is: 4
2024-11-13T16:06:30,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:06:30,798 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT
2024-11-13T16:06:30,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741835_1011 (size=381)
2024-11-13T16:06:30,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741835_1011 (size=381)
2024-11-13T16:06:30,807 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9cf1f5faf3b84cb9bdad9af6f5b3801d, NAME => 'TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b
2024-11-13T16:06:30,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741836_1012 (size=64)
2024-11-13T16:06:30,812 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741836_1012 (size=64)
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 9cf1f5faf3b84cb9bdad9af6f5b3801d, disabling compactions & flushes
2024-11-13T16:06:30,813 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d. after waiting 0 ms
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:30,813 INFO  [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:30,813 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:
Waiting for close lock at 1731513990813Disabling compacts and flushes for region at 1731513990813Disabling writes for close at 1731513990813Writing region close event to WAL at 1731513990813Closed at 1731513990813
2024-11-13T16:06:30,814 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META
2024-11-13T16:06:30,815 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731513990814"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731513990814"}]},"ts":"1731513990814"}
2024-11-13T16:06:30,817 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta.
2024-11-13T16:06:30,818 INFO  [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS
2024-11-13T16:06:30,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513990818"}]},"ts":"1731513990818"}
2024-11-13T16:06:30,819 INFO  [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta
2024-11-13T16:06:30,820 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, ASSIGN}]
2024-11-13T16:06:30,821 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, ASSIGN
2024-11-13T16:06:30,822 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, ASSIGN; state=OFFLINE, location=c4c650e80f58,44287,1731513989865; forceNewPlan=false, retain=false
2024-11-13T16:06:30,972 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cf1f5faf3b84cb9bdad9af6f5b3801d, regionState=OPENING, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:30,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, ASSIGN because future has completed
2024-11-13T16:06:30,975 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865}]
2024-11-13T16:06:31,132 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:31,132 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9cf1f5faf3b84cb9bdad9af6f5b3801d, NAME => 'TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:06:31,132 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,132 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:31,132 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,132 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,133 INFO  [StoreOpener-9cf1f5faf3b84cb9bdad9af6f5b3801d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9cf1f5faf3b84cb9bdad9af6f5b3801d 
2024-11-13T16:06:31,134 INFO  [StoreOpener-9cf1f5faf3b84cb9bdad9af6f5b3801d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9cf1f5faf3b84cb9bdad9af6f5b3801d columnFamilyName info
2024-11-13T16:06:31,135 DEBUG [StoreOpener-9cf1f5faf3b84cb9bdad9af6f5b3801d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:31,135 INFO  [StoreOpener-9cf1f5faf3b84cb9bdad9af6f5b3801d-1 {}] regionserver.HStore(327): Store=9cf1f5faf3b84cb9bdad9af6f5b3801d/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:31,135 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,136 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,136 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,136 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,136 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,138 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,139 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:06:31,140 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9cf1f5faf3b84cb9bdad9af6f5b3801d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=689264, jitterRate=-0.12355634570121765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:06:31,140 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:31,140 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:
Running coprocessor pre-open hook at 1731513991132Writing region info on filesystem at 1731513991132Initializing all the Stores at 1731513991133 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731513991133Cleaning up temporary data from old regions at 1731513991136 (+3 ms)Running coprocessor post-open hooks at 1731513991140 (+4 ms)Region opened successfully at 1731513991140
2024-11-13T16:06:31,142 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., pid=6, masterSystemTime=1731513991128
2024-11-13T16:06:31,144 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:31,144 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:31,145 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9cf1f5faf3b84cb9bdad9af6f5b3801d, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:31,147 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865 because future has completed
2024-11-13T16:06:31,151 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5
2024-11-13T16:06:31,151 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865 in 173 msec
2024-11-13T16:06:31,154 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4
2024-11-13T16:06:31,154 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, ASSIGN in 331 msec
2024-11-13T16:06:31,155 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE
2024-11-13T16:06:31,155 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731513991155"}]},"ts":"1731513991155"}
2024-11-13T16:06:31,157 INFO  [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta
2024-11-13T16:06:31,158 INFO  [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION
2024-11-13T16:06:31,160 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 363 msec
2024-11-13T16:06:31,562 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:31,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:32,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:32,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:33,563 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:33,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:33,647 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,647 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,648 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,663 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,664 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,666 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,666 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,667 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:33,668 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,172 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:06:34,173 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,174 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,175 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,193 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,194 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,196 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,197 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,197 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,199 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:34,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:34,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:35,564 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:35,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:36,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta'
2024-11-13T16:06:36,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling'
2024-11-13T16:06:36,370 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling
2024-11-13T16:06:36,370 INFO  [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer
2024-11-13T16:06:36,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling
2024-11-13T16:06:36,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:36,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:37,565 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:37,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:38,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:38,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:39,566 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:39,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:40,567 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:40,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4
2024-11-13T16:06:40,894 INFO  [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed
2024-11-13T16:06:40,894 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100
2024-11-13T16:06:40,897 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling
2024-11-13T16:06:40,897 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:40,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2]
2024-11-13T16:06:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:40,911 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cf1f5faf3b84cb9bdad9af6f5b3801d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:06:40,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/686a2c100a0c4914bb06c255d91f203d is 1080, key is row0001/info:/1731514000900/Put/seqid=0
2024-11-13T16:06:40,934 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741837_1013 (size=12509)
2024-11-13T16:06:40,935 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741837_1013 (size=12509)
2024-11-13T16:06:40,935 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/686a2c100a0c4914bb06c255d91f203d
2024-11-13T16:06:40,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/686a2c100a0c4914bb06c255d91f203d as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d
2024-11-13T16:06:40,946 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-13T16:06:40,947 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d, entries=7, sequenceid=11, filesize=12.2 K
2024-11-13T16:06:40,948 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 9cf1f5faf3b84cb9bdad9af6f5b3801d in 38ms, sequenceid=11, compaction requested=false
2024-11-13T16:06:40,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:33068 deadline: 1731514010945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:06:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:06:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 because the exception is null or not the one we care about
2024-11-13T16:06:41,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:41,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:41,873 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:06:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,874 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,875 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,876 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,900 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,901 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,902 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,902 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,907 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,908 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:41,911 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:42,568 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:42,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:43,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:43,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:44,569 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:44,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:45,570 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:45,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:46,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:46,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:47,571 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:47,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:48,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:48,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:49,572 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:49,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:50,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:50,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:51,026 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cf1f5faf3b84cb9bdad9af6f5b3801d 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB
2024-11-13T16:06:51,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f85831a63def4ac991c30dc650a6beba is 1080, key is row0008/info:/1731514000911/Put/seqid=0
2024-11-13T16:06:51,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741838_1014 (size=29761)
2024-11-13T16:06:51,035 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741838_1014 (size=29761)
2024-11-13T16:06:51,036 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f85831a63def4ac991c30dc650a6beba
2024-11-13T16:06:51,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f85831a63def4ac991c30dc650a6beba as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba
2024-11-13T16:06:51,046 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba, entries=23, sequenceid=37, filesize=29.1 K
2024-11-13T16:06:51,047 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 9cf1f5faf3b84cb9bdad9af6f5b3801d in 22ms, sequenceid=37, compaction requested=false
2024-11-13T16:06:51,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:51,048 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K
2024-11-13T16:06:51,048 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:51,048 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba because midkey is the same as first or last row
2024-11-13T16:06:51,573 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:51,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:52,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:52,574 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:53,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,038 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cf1f5faf3b84cb9bdad9af6f5b3801d 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:06:53,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f69eaa27dde74c49960bdce9a80d8096 is 1080, key is row0031/info:/1731514011027/Put/seqid=0
2024-11-13T16:06:53,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741839_1015 (size=12509)
2024-11-13T16:06:53,048 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741839_1015 (size=12509)
2024-11-13T16:06:53,048 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f69eaa27dde74c49960bdce9a80d8096
2024-11-13T16:06:53,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/f69eaa27dde74c49960bdce9a80d8096 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096
2024-11-13T16:06:53,059 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096, entries=7, sequenceid=47, filesize=12.2 K
2024-11-13T16:06:53,060 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 9cf1f5faf3b84cb9bdad9af6f5b3801d in 22ms, sequenceid=47, compaction requested=true
2024-11-13T16:06:53,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:53,060 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,060 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,060 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba because midkey is the same as first or last row
2024-11-13T16:06:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cf1f5faf3b84cb9bdad9af6f5b3801d:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:06:53,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,061 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:06:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,062 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cf1f5faf3b84cb9bdad9af6f5b3801d 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-13T16:06:53,062 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:06:53,062 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9cf1f5faf3b84cb9bdad9af6f5b3801d/info is initiating minor compaction (all files)
2024-11-13T16:06:53,062 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cf1f5faf3b84cb9bdad9af6f5b3801d/info in TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,062 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp, totalSize=53.5 K
2024-11-13T16:06:53,063 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 686a2c100a0c4914bb06c255d91f203d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731514000900
2024-11-13T16:06:53,063 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85831a63def4ac991c30dc650a6beba, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731514000911
2024-11-13T16:06:53,064 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting f69eaa27dde74c49960bdce9a80d8096, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731514011027
2024-11-13T16:06:53,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/78959e8fdefd466897eecb5701dd2de0 is 1080, key is row0038/info:/1731514013039/Put/seqid=0
2024-11-13T16:06:53,070 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741840_1016 (size=20064)
2024-11-13T16:06:53,071 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741840_1016 (size=20064)
2024-11-13T16:06:53,071 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/78959e8fdefd466897eecb5701dd2de0
2024-11-13T16:06:53,078 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cf1f5faf3b84cb9bdad9af6f5b3801d#info#compaction#61 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:06:53,078 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/70c731d61dbd43eaa3bc0a298939bb89 is 1080, key is row0001/info:/1731514000900/Put/seqid=0
2024-11-13T16:06:53,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/78959e8fdefd466897eecb5701dd2de0 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0
2024-11-13T16:06:53,085 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0, entries=14, sequenceid=64, filesize=19.6 K
2024-11-13T16:06:53,086 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 9cf1f5faf3b84cb9bdad9af6f5b3801d in 25ms, sequenceid=64, compaction requested=false
2024-11-13T16:06:53,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:53,086 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=73.1 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,086 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,086 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba because midkey is the same as first or last row
2024-11-13T16:06:53,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,088 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9cf1f5faf3b84cb9bdad9af6f5b3801d 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-13T16:06:53,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741841_1017 (size=44978)
2024-11-13T16:06:53,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741841_1017 (size=44978)
2024-11-13T16:06:53,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/7e4bb0f83ae74d7587a8c22d7ad84855 is 1080, key is row0052/info:/1731514013062/Put/seqid=0
2024-11-13T16:06:53,095 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/70c731d61dbd43eaa3bc0a298939bb89 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89
2024-11-13T16:06:53,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741842_1018 (size=18987)
2024-11-13T16:06:53,098 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741842_1018 (size=18987)
2024-11-13T16:06:53,099 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/7e4bb0f83ae74d7587a8c22d7ad84855
2024-11-13T16:06:53,102 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cf1f5faf3b84cb9bdad9af6f5b3801d/info of 9cf1f5faf3b84cb9bdad9af6f5b3801d into 70c731d61dbd43eaa3bc0a298939bb89(size=43.9 K), total size for store is 63.5 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:53,103 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., storeName=9cf1f5faf3b84cb9bdad9af6f5b3801d/info, priority=13, startTime=1731514013060; duration=0sec
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89 because midkey is the same as first or last row
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89 because midkey is the same as first or last row
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=63.5 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89 because midkey is the same as first or last row
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,103 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cf1f5faf3b84cb9bdad9af6f5b3801d:info
2024-11-13T16:06:53,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/7e4bb0f83ae74d7587a8c22d7ad84855 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855
2024-11-13T16:06:53,109 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855, entries=13, sequenceid=80, filesize=18.5 K
2024-11-13T16:06:53,110 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 9cf1f5faf3b84cb9bdad9af6f5b3801d in 22ms, sequenceid=80, compaction requested=true
2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.1 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89 because midkey is the same as first or last row
2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9cf1f5faf3b84cb9bdad9af6f5b3801d:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:06:53,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,110 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:06:53,111 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 84029 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:06:53,111 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9cf1f5faf3b84cb9bdad9af6f5b3801d/info is initiating minor compaction (all files)
2024-11-13T16:06:53,112 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9cf1f5faf3b84cb9bdad9af6f5b3801d/info in TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,112 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp, totalSize=82.1 K
2024-11-13T16:06:53,112 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c731d61dbd43eaa3bc0a298939bb89, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731514000900
2024-11-13T16:06:53,113 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78959e8fdefd466897eecb5701dd2de0, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731514013039
2024-11-13T16:06:53,113 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e4bb0f83ae74d7587a8c22d7ad84855, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731514013062
2024-11-13T16:06:53,124 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9cf1f5faf3b84cb9bdad9af6f5b3801d#info#compaction#63 average throughput is 32.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:06:53,124 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/a01a6a23f9174cb9a00e068d3593e086 is 1080, key is row0001/info:/1731514000900/Put/seqid=0
2024-11-13T16:06:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741843_1019 (size=74301)
2024-11-13T16:06:53,128 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741843_1019 (size=74301)
2024-11-13T16:06:53,135 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/.tmp/info/a01a6a23f9174cb9a00e068d3593e086 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086
2024-11-13T16:06:53,141 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9cf1f5faf3b84cb9bdad9af6f5b3801d/info of 9cf1f5faf3b84cb9bdad9af6f5b3801d into a01a6a23f9174cb9a00e068d3593e086(size=72.6 K), total size for store is 72.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:06:53,141 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:

2024-11-13T16:06:53,141 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., storeName=9cf1f5faf3b84cb9bdad9af6f5b3801d/info, priority=13, startTime=1731514013110; duration=0sec
2024-11-13T16:06:53,141 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,141 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,142 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,142 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,142 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K
2024-11-13T16:06:53,142 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1
2024-11-13T16:06:53,143 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,143 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,143 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9cf1f5faf3b84cb9bdad9af6f5b3801d:info
2024-11-13T16:06:53,144 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41807 {}] assignment.AssignmentManager(1355): Split request from c4c650e80f58,44287,1731513989865, parent={ENCODED => 9cf1f5faf3b84cb9bdad9af6f5b3801d, NAME => 'TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062
2024-11-13T16:06:53,149 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41807 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,153 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41807 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9cf1f5faf3b84cb9bdad9af6f5b3801d, daughterA=e83fccdcd88e4fbfaa390db8e2bf57d8, daughterB=9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,154 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9cf1f5faf3b84cb9bdad9af6f5b3801d, daughterA=e83fccdcd88e4fbfaa390db8e2bf57d8, daughterB=9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,154 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9cf1f5faf3b84cb9bdad9af6f5b3801d, daughterA=e83fccdcd88e4fbfaa390db8e2bf57d8, daughterB=9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,154 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9cf1f5faf3b84cb9bdad9af6f5b3801d, daughterA=e83fccdcd88e4fbfaa390db8e2bf57d8, daughterB=9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,163 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, UNASSIGN}]
2024-11-13T16:06:53,164 INFO  [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, UNASSIGN
2024-11-13T16:06:53,166 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9cf1f5faf3b84cb9bdad9af6f5b3801d, regionState=CLOSING, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, UNASSIGN because future has completed
2024-11-13T16:06:53,169 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false
2024-11-13T16:06:53,169 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865}]
2024-11-13T16:06:53,326 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,326 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true
2024-11-13T16:06:53,327 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 9cf1f5faf3b84cb9bdad9af6f5b3801d, disabling compactions & flushes
2024-11-13T16:06:53,327 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,327 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,327 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d. after waiting 0 ms
2024-11-13T16:06:53,327 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,328 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855] to archive
2024-11-13T16:06:53,329 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:06:53,331 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/686a2c100a0c4914bb06c255d91f203d
2024-11-13T16:06:53,333 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f85831a63def4ac991c30dc650a6beba
2024-11-13T16:06:53,334 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/70c731d61dbd43eaa3bc0a298939bb89
2024-11-13T16:06:53,335 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/f69eaa27dde74c49960bdce9a80d8096
2024-11-13T16:06:53,336 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/78959e8fdefd466897eecb5701dd2de0
2024-11-13T16:06:53,337 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/7e4bb0f83ae74d7587a8c22d7ad84855
2024-11-13T16:06:53,343 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=1
2024-11-13T16:06:53,343 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.
2024-11-13T16:06:53,343 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 9cf1f5faf3b84cb9bdad9af6f5b3801d:
Waiting for close lock at 1731514013327Running coprocessor pre-close hooks at 1731514013327Disabling compacts and flushes for region at 1731514013327Disabling writes for close at 1731514013327Writing region close event to WAL at 1731514013339 (+12 ms)Running coprocessor post-close hooks at 1731514013343 (+4 ms)Closed at 1731514013343
2024-11-13T16:06:53,346 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,346 INFO  [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9cf1f5faf3b84cb9bdad9af6f5b3801d, regionState=CLOSED
2024-11-13T16:06:53,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865 because future has completed
2024-11-13T16:06:53,352 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8
2024-11-13T16:06:53,352 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 9cf1f5faf3b84cb9bdad9af6f5b3801d, server=c4c650e80f58,44287,1731513989865 in 181 msec
2024-11-13T16:06:53,354 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7
2024-11-13T16:06:53,354 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, UNASSIGN in 190 msec
2024-11-13T16:06:53,361 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:53,363 INFO  [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 1 storefiles, region=9cf1f5faf3b84cb9bdad9af6f5b3801d, threads=1
2024-11-13T16:06:53,365 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086 for region: 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741844_1020 (size=27)
2024-11-13T16:06:53,377 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741844_1020 (size=27)
2024-11-13T16:06:53,386 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741845_1021 (size=27)
2024-11-13T16:06:53,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741845_1021 (size=27)
2024-11-13T16:06:53,388 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086 for region: 9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:06:53,389 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 9cf1f5faf3b84cb9bdad9af6f5b3801d Daughter A: [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d] storefiles, Daughter B: [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d] storefiles.
2024-11-13T16:06:53,396 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741846_1022 (size=71)
2024-11-13T16:06:53,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741846_1022 (size=71)
2024-11-13T16:06:53,399 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:53,409 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741847_1023 (size=71)
2024-11-13T16:06:53,410 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741847_1023 (size=71)
2024-11-13T16:06:53,411 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:53,419 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-11-13T16:06:53,421 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1
2024-11-13T16:06:53,423 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731514013423"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731514013423"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731514013423"}]},"ts":"1731514013423"}
2024-11-13T16:06:53,423 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731514013423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514013423"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731514013423"}]},"ts":"1731514013423"}
2024-11-13T16:06:53,423 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731514013423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731514013423"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731514013423"}]},"ts":"1731514013423"}
2024-11-13T16:06:53,440 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e83fccdcd88e4fbfaa390db8e2bf57d8, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d56dfa1976de842f0efb20587b93a2b, ASSIGN}]
2024-11-13T16:06:53,441 INFO  [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e83fccdcd88e4fbfaa390db8e2bf57d8, ASSIGN
2024-11-13T16:06:53,441 INFO  [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d56dfa1976de842f0efb20587b93a2b, ASSIGN
2024-11-13T16:06:53,442 INFO  [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e83fccdcd88e4fbfaa390db8e2bf57d8, ASSIGN; state=SPLITTING_NEW, location=c4c650e80f58,44287,1731513989865; forceNewPlan=false, retain=false
2024-11-13T16:06:53,442 INFO  [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d56dfa1976de842f0efb20587b93a2b, ASSIGN; state=SPLITTING_NEW, location=c4c650e80f58,44287,1731513989865; forceNewPlan=false, retain=false
2024-11-13T16:06:53,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:53,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:53,592 INFO  [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9d56dfa1976de842f0efb20587b93a2b, regionState=OPENING, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,592 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e83fccdcd88e4fbfaa390db8e2bf57d8, regionState=OPENING, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d56dfa1976de842f0efb20587b93a2b, ASSIGN because future has completed
2024-11-13T16:06:53,595 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865}]
2024-11-13T16:06:53,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e83fccdcd88e4fbfaa390db8e2bf57d8, ASSIGN because future has completed
2024-11-13T16:06:53,596 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e83fccdcd88e4fbfaa390db8e2bf57d8, server=c4c650e80f58,44287,1731513989865}]
2024-11-13T16:06:53,751 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:06:53,751 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d56dfa1976de842f0efb20587b93a2b, NAME => 'TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.', STARTKEY => 'row0062', ENDKEY => ''}
2024-11-13T16:06:53,751 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,751 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:53,751 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,751 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,753 INFO  [StoreOpener-9d56dfa1976de842f0efb20587b93a2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 9d56dfa1976de842f0efb20587b93a2b 
2024-11-13T16:06:53,753 INFO  [StoreOpener-9d56dfa1976de842f0efb20587b93a2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d56dfa1976de842f0efb20587b93a2b columnFamilyName info
2024-11-13T16:06:53,754 DEBUG [StoreOpener-9d56dfa1976de842f0efb20587b93a2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:53,762 DEBUG [StoreOpener-9d56dfa1976de842f0efb20587b93a2b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-top
2024-11-13T16:06:53,762 INFO  [StoreOpener-9d56dfa1976de842f0efb20587b93a2b-1 {}] regionserver.HStore(327): Store=9d56dfa1976de842f0efb20587b93a2b/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:53,762 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,763 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,764 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,764 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,764 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,766 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,766 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 9d56dfa1976de842f0efb20587b93a2b; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=824504, jitterRate=0.0484122633934021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:06:53,766 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:06:53,767 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 9d56dfa1976de842f0efb20587b93a2b:
Running coprocessor pre-open hook at 1731514013752Writing region info on filesystem at 1731514013752Initializing all the Stores at 1731514013752Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514013752Cleaning up temporary data from old regions at 1731514013764 (+12 ms)Running coprocessor post-open hooks at 1731514013766 (+2 ms)Region opened successfully at 1731514013767 (+1 ms)
2024-11-13T16:06:53,768 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., pid=12, masterSystemTime=1731514013747
2024-11-13T16:06:53,768 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:06:53,768 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,768 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-13T16:06:53,768 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:06:53,768 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:06:53,768 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:06:53,769 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-top] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=72.6 K
2024-11-13T16:06:53,769 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731514000900
2024-11-13T16:06:53,770 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:06:53,770 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:06:53,770 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:06:53,770 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => e83fccdcd88e4fbfaa390db8e2bf57d8, NAME => 'TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.', STARTKEY => '', ENDKEY => 'row0062'}
2024-11-13T16:06:53,771 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,771 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:06:53,771 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,771 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,771 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=9d56dfa1976de842f0efb20587b93a2b, regionState=OPEN, openSeqNum=86, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,772 INFO  [StoreOpener-e83fccdcd88e4fbfaa390db8e2bf57d8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e83fccdcd88e4fbfaa390db8e2bf57d8 
2024-11-13T16:06:53,773 INFO  [StoreOpener-e83fccdcd88e4fbfaa390db8e2bf57d8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e83fccdcd88e4fbfaa390db8e2bf57d8 columnFamilyName info
2024-11-13T16:06:53,773 DEBUG [StoreOpener-e83fccdcd88e4fbfaa390db8e2bf57d8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:06:53,773 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 1588230740
2024-11-13T16:06:53,773 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all.
2024-11-13T16:06:53,773 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB
2024-11-13T16:06:53,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865 because future has completed
2024-11-13T16:06:53,777 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11
2024-11-13T16:06:53,777 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865 in 180 msec
2024-11-13T16:06:53,781 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=9d56dfa1976de842f0efb20587b93a2b, ASSIGN in 337 msec
2024-11-13T16:06:53,782 DEBUG [StoreOpener-e83fccdcd88e4fbfaa390db8e2bf57d8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-bottom
2024-11-13T16:06:53,783 INFO  [StoreOpener-e83fccdcd88e4fbfaa390db8e2bf57d8-1 {}] regionserver.HStore(327): Store=e83fccdcd88e4fbfaa390db8e2bf57d8/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:06:53,783 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,784 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,785 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,785 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,785 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,787 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,788 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened e83fccdcd88e4fbfaa390db8e2bf57d8; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=852735, jitterRate=0.08430896699428558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1}
2024-11-13T16:06:53,788 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:06:53,788 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for e83fccdcd88e4fbfaa390db8e2bf57d8:
Running coprocessor pre-open hook at 1731514013771Writing region info on filesystem at 1731514013771Initializing all the Stores at 1731514013771Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514013771Cleaning up temporary data from old regions at 1731514013785 (+14 ms)Running coprocessor post-open hooks at 1731514013788 (+3 ms)Region opened successfully at 1731514013788
2024-11-13T16:06:53,789 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8., pid=13, masterSystemTime=1731514013747
2024-11-13T16:06:53,789 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store e83fccdcd88e4fbfaa390db8e2bf57d8:info, priority=-2147483648, current under compaction store size is 2
2024-11-13T16:06:53,789 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,789 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking
2024-11-13T16:06:53,790 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#64 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:06:53,790 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:06:53,790 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HStore(1541): e83fccdcd88e4fbfaa390db8e2bf57d8/info is initiating minor compaction (all files)
2024-11-13T16:06:53,790 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e83fccdcd88e4fbfaa390db8e2bf57d8/info in TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:06:53,790 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-bottom] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/.tmp, totalSize=72.6 K
2024-11-13T16:06:53,790 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/42ec8b402afd4a1680151e23f5a212b7 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:06:53,791 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] compactions.Compactor(225): Compacting a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731514000900
2024-11-13T16:06:53,792 DEBUG [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:06:53,792 INFO  [RS_OPEN_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:06:53,793 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=e83fccdcd88e4fbfaa390db8e2bf57d8, regionState=OPEN, openSeqNum=86, regionLocation=c4c650e80f58,44287,1731513989865
2024-11-13T16:06:53,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/aef1e3f27a2c49b69c0895bf7de2cb41 is 193, key is TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b./info:regioninfo/1731514013771/Put/seqid=0
2024-11-13T16:06:53,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure e83fccdcd88e4fbfaa390db8e2bf57d8, server=c4c650e80f58,44287,1731513989865 because future has completed
2024-11-13T16:06:53,800 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741848_1024 (size=8260)
2024-11-13T16:06:53,801 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741848_1024 (size=8260)
2024-11-13T16:06:53,804 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741849_1025 (size=9882)
2024-11-13T16:06:53,805 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/aef1e3f27a2c49b69c0895bf7de2cb41
2024-11-13T16:06:53,805 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741849_1025 (size=9882)
2024-11-13T16:06:53,806 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10
2024-11-13T16:06:53,806 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure e83fccdcd88e4fbfaa390db8e2bf57d8, server=c4c650e80f58,44287,1731513989865 in 207 msec
2024-11-13T16:06:53,808 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7
2024-11-13T16:06:53,809 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=e83fccdcd88e4fbfaa390db8e2bf57d8, ASSIGN in 366 msec
2024-11-13T16:06:53,809 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/42ec8b402afd4a1680151e23f5a212b7 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/42ec8b402afd4a1680151e23f5a212b7
2024-11-13T16:06:53,811 INFO  [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=9cf1f5faf3b84cb9bdad9af6f5b3801d, daughterA=e83fccdcd88e4fbfaa390db8e2bf57d8, daughterB=9d56dfa1976de842f0efb20587b93a2b in 660 msec
2024-11-13T16:06:53,815 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into 42ec8b402afd4a1680151e23f5a212b7(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:06:53,815 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:06:53,815 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=15, startTime=1731514013768; duration=0sec
2024-11-13T16:06:53,815 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,815 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:06:53,821 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e83fccdcd88e4fbfaa390db8e2bf57d8#info#compaction#66 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:06:53,822 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/.tmp/info/29fea0a0ee27455e8c93a50b5899866d is 1080, key is row0001/info:/1731514000900/Put/seqid=0
2024-11-13T16:06:53,826 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741850_1026 (size=70862)
2024-11-13T16:06:53,827 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741850_1026 (size=70862)
2024-11-13T16:06:53,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/ns/567f4bdc884f4699bb6986b173be7e7f is 43, key is default/ns:d/1731513990680/Put/seqid=0
2024-11-13T16:06:53,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741851_1027 (size=5153)
2024-11-13T16:06:53,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741851_1027 (size=5153)
2024-11-13T16:06:53,833 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/ns/567f4bdc884f4699bb6986b173be7e7f
2024-11-13T16:06:53,833 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/.tmp/info/29fea0a0ee27455e8c93a50b5899866d as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/29fea0a0ee27455e8c93a50b5899866d
2024-11-13T16:06:53,839 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in e83fccdcd88e4fbfaa390db8e2bf57d8/info of e83fccdcd88e4fbfaa390db8e2bf57d8 into 29fea0a0ee27455e8c93a50b5899866d(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:06:53,839 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e83fccdcd88e4fbfaa390db8e2bf57d8:

2024-11-13T16:06:53,839 INFO  [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8., storeName=e83fccdcd88e4fbfaa390db8e2bf57d8/info, priority=15, startTime=1731514013789; duration=0sec
2024-11-13T16:06:53,839 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:06:53,839 DEBUG [RS:0;c4c650e80f58:44287-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e83fccdcd88e4fbfaa390db8e2bf57d8:info
2024-11-13T16:06:53,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/table/22dee824b5644ca0b52cd787a0e085d7 is 65, key is TestLogRolling-testLogRolling/table:state/1731513991155/Put/seqid=0
2024-11-13T16:06:53,856 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741852_1028 (size=5340)
2024-11-13T16:06:53,857 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741852_1028 (size=5340)
2024-11-13T16:06:53,857 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/table/22dee824b5644ca0b52cd787a0e085d7
2024-11-13T16:06:53,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/aef1e3f27a2c49b69c0895bf7de2cb41 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/info/aef1e3f27a2c49b69c0895bf7de2cb41
2024-11-13T16:06:53,867 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/info/aef1e3f27a2c49b69c0895bf7de2cb41, entries=30, sequenceid=17, filesize=9.7 K
2024-11-13T16:06:53,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/ns/567f4bdc884f4699bb6986b173be7e7f as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/ns/567f4bdc884f4699bb6986b173be7e7f
2024-11-13T16:06:53,872 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/ns/567f4bdc884f4699bb6986b173be7e7f, entries=2, sequenceid=17, filesize=5.0 K
2024-11-13T16:06:53,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/table/22dee824b5644ca0b52cd787a0e085d7 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/table/22dee824b5644ca0b52cd787a0e085d7
2024-11-13T16:06:53,877 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/table/22dee824b5644ca0b52cd787a0e085d7, entries=2, sequenceid=17, filesize=5.2 K
2024-11-13T16:06:53,878 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 105ms, sequenceid=17, compaction requested=false
2024-11-13T16:06:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-13T16:06:54,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:54,575 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:33068 deadline: 1731514025089, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d. is not online on c4c650e80f58,44287,1731513989865
2024-11-13T16:06:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d. is not online on c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:06:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d. is not online on c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186)
	at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:06:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731513990794.9cf1f5faf3b84cb9bdad9af6f5b3801d., hostname=c4c650e80f58,44287,1731513989865, seqNum=2 from cache
2024-11-13T16:06:55,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:55,576 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:56,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:56,577 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:57,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:57,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:58,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,344 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,345 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,370 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,371 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,376 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,377 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,380 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:58,578 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:58,889 WARN  [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,890 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,891 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,923 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,924 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,925 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,930 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:58,934 WARN  [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null
2024-11-13T16:06:59,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:59,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:06:59,807 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details.
2024-11-13T16:07:00,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:00,579 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:01,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:01,580 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:02,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:02,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:03,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:03,581 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:04,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:04,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., hostname=c4c650e80f58,44287,1731513989865, seqNum=86]
2024-11-13T16:07:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:05,150 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:07:05,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/08a5511b76404359bdf45e697b39e14d is 1080, key is row0065/info:/1731514025141/Put/seqid=0
2024-11-13T16:07:05,160 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741853_1029 (size=12509)
2024-11-13T16:07:05,161 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741853_1029 (size=12509)
2024-11-13T16:07:05,161 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/08a5511b76404359bdf45e697b39e14d
2024-11-13T16:07:05,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/08a5511b76404359bdf45e697b39e14d as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d
2024-11-13T16:07:05,172 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d, entries=7, sequenceid=96, filesize=12.2 K
2024-11-13T16:07:05,173 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 9d56dfa1976de842f0efb20587b93a2b in 23ms, sequenceid=96, compaction requested=false
2024-11-13T16:07:05,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:05,174 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-13T16:07:05,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ddf29ade2fe94133a92dd79a46dd529b is 1080, key is row0072/info:/1731514025151/Put/seqid=0
2024-11-13T16:07:05,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741854_1030 (size=22222)
2024-11-13T16:07:05,182 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741854_1030 (size=22222)
2024-11-13T16:07:05,183 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ddf29ade2fe94133a92dd79a46dd529b
2024-11-13T16:07:05,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ddf29ade2fe94133a92dd79a46dd529b as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b
2024-11-13T16:07:05,193 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b, entries=16, sequenceid=115, filesize=21.7 K
2024-11-13T16:07:05,194 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=9.46 KB/9684 for 9d56dfa1976de842f0efb20587b93a2b in 20ms, sequenceid=115, compaction requested=true
2024-11-13T16:07:05,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:05,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:05,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:05,194 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:05,195 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 42991 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:05,195 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:05,195 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:05,195 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/42ec8b402afd4a1680151e23f5a212b7, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=42.0 K
2024-11-13T16:07:05,196 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42ec8b402afd4a1680151e23f5a212b7, keycount=3, bloomtype=ROW, size=8.1 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731514013083
2024-11-13T16:07:05,196 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08a5511b76404359bdf45e697b39e14d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731514025141
2024-11-13T16:07:05,196 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf29ade2fe94133a92dd79a46dd529b, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731514025151
2024-11-13T16:07:05,206 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#71 average throughput is 26.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:05,206 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/64ddf4b4b74c4926aab26cd368e5f82a is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:05,211 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741855_1031 (size=33163)
2024-11-13T16:07:05,212 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741855_1031 (size=33163)
2024-11-13T16:07:05,216 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/64ddf4b4b74c4926aab26cd368e5f82a as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/64ddf4b4b74c4926aab26cd368e5f82a
2024-11-13T16:07:05,221 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into 64ddf4b4b74c4926aab26cd368e5f82a(size=32.4 K), total size for store is 32.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:05,221 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:05,221 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514025194; duration=0sec
2024-11-13T16:07:05,221 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:05,221 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:05,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:05,582 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:06,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:06,583 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:07,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:07,191 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB
2024-11-13T16:07:07,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/62f914f2a2c5452bb736320143e74197 is 1080, key is row0088/info:/1731514025175/Put/seqid=0
2024-11-13T16:07:07,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741856_1032 (size=15741)
2024-11-13T16:07:07,202 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741856_1032 (size=15741)
2024-11-13T16:07:07,202 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/62f914f2a2c5452bb736320143e74197
2024-11-13T16:07:07,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/62f914f2a2c5452bb736320143e74197 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197
2024-11-13T16:07:07,214 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197, entries=10, sequenceid=129, filesize=15.4 K
2024-11-13T16:07:07,215 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10760, heapSize ~11.48 KB/11760, currentSize=15.76 KB/16140 for 9d56dfa1976de842f0efb20587b93a2b in 24ms, sequenceid=129, compaction requested=false
2024-11-13T16:07:07,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:07,216 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-13T16:07:07,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d8b8aa1f0121452e966abf888424147d is 1080, key is row0098/info:/1731514027192/Put/seqid=0
2024-11-13T16:07:07,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741857_1033 (size=22238)
2024-11-13T16:07:07,226 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741857_1033 (size=22238)
2024-11-13T16:07:07,236 WARN  [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit.
org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?]
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT]
2024-11-13T16:07:07,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.3:33068 deadline: 1731514037236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865
2024-11-13T16:07:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., hostname=c4c650e80f58,44287,1731513989865, seqNum=86 , the old value is region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., hostname=c4c650e80f58,44287,1731513989865, seqNum=86, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:07:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., hostname=c4c650e80f58,44287,1731513989865, seqNum=86 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=9d56dfa1976de842f0efb20587b93a2b, server=c4c650e80f58,44287,1731513989865
	at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308)
	at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329)
	at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216)
	at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018)
	at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981)
	at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506)
	at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444)
	at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102)
	at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82)

2024-11-13T16:07:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., hostname=c4c650e80f58,44287,1731513989865, seqNum=86 because the exception is null or not the one we care about
2024-11-13T16:07:07,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:07,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:07,627 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d8b8aa1f0121452e966abf888424147d
2024-11-13T16:07:07,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d8b8aa1f0121452e966abf888424147d as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d
2024-11-13T16:07:07,637 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d, entries=16, sequenceid=148, filesize=21.7 K
2024-11-13T16:07:07,638 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for 9d56dfa1976de842f0efb20587b93a2b in 422ms, sequenceid=148, compaction requested=true
2024-11-13T16:07:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:07,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:07,638 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:07,639 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 71142 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:07,639 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:07,639 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:07,640 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/64ddf4b4b74c4926aab26cd368e5f82a, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=69.5 K
2024-11-13T16:07:07,640 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64ddf4b4b74c4926aab26cd368e5f82a, keycount=26, bloomtype=ROW, size=32.4 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731514013083
2024-11-13T16:07:07,640 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f914f2a2c5452bb736320143e74197, keycount=10, bloomtype=ROW, size=15.4 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731514025175
2024-11-13T16:07:07,641 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8b8aa1f0121452e966abf888424147d, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731514027192
2024-11-13T16:07:07,649 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#74 average throughput is 53.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:07,649 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c3885e57a98147d18be4e11ee3474022 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:07,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741858_1034 (size=61324)
2024-11-13T16:07:07,654 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741858_1034 (size=61324)
2024-11-13T16:07:07,660 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c3885e57a98147d18be4e11ee3474022 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c3885e57a98147d18be4e11ee3474022
2024-11-13T16:07:07,666 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into c3885e57a98147d18be4e11ee3474022(size=59.9 K), total size for store is 59.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:07,666 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:07,666 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514027638; duration=0sec
2024-11-13T16:07:07,666 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:07,666 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:08,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:08,584 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:09,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:09,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:10,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:10,585 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:10,697 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker!
2024-11-13T16:07:10,697 INFO  [master/c4c650e80f58:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore!
2024-11-13T16:07:11,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:11,586 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:12,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:12,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:13,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:13,587 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:14,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:14,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:15,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:15,588 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:15,650 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375
2024-11-13T16:07:16,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:16,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:17,255 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-13T16:07:17,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/0556e5ab58a54dc6ab08c24e50063a68 is 1080, key is row0114/info:/1731514027217/Put/seqid=0
2024-11-13T16:07:17,263 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741859_1035 (size=20078)
2024-11-13T16:07:17,264 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741859_1035 (size=20078)
2024-11-13T16:07:17,264 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/0556e5ab58a54dc6ab08c24e50063a68
2024-11-13T16:07:17,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/0556e5ab58a54dc6ab08c24e50063a68 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68
2024-11-13T16:07:17,273 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68, entries=14, sequenceid=166, filesize=19.6 K
2024-11-13T16:07:17,274 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=1.05 KB/1076 for 9d56dfa1976de842f0efb20587b93a2b in 19ms, sequenceid=166, compaction requested=false
2024-11-13T16:07:17,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:17,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:17,589 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:18,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:18,590 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:19,267 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:07:19,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/1749449e03af4bc0855bce7641a0ffce is 1080, key is row0128/info:/1731514037256/Put/seqid=0
2024-11-13T16:07:19,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741860_1036 (size=12516)
2024-11-13T16:07:19,279 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741860_1036 (size=12516)
2024-11-13T16:07:19,280 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/1749449e03af4bc0855bce7641a0ffce
2024-11-13T16:07:19,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/1749449e03af4bc0855bce7641a0ffce as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce
2024-11-13T16:07:19,290 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce, entries=7, sequenceid=176, filesize=12.2 K
2024-11-13T16:07:19,291 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for 9d56dfa1976de842f0efb20587b93a2b in 24ms, sequenceid=176, compaction requested=true
2024-11-13T16:07:19,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:19,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:19,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:19,291 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:19,292 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 93918 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:19,292 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:19,292 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB
2024-11-13T16:07:19,292 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:19,292 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c3885e57a98147d18be4e11ee3474022, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=91.7 K
2024-11-13T16:07:19,293 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3885e57a98147d18be4e11ee3474022, keycount=52, bloomtype=ROW, size=59.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731514013083
2024-11-13T16:07:19,293 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0556e5ab58a54dc6ab08c24e50063a68, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731514027217
2024-11-13T16:07:19,294 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1749449e03af4bc0855bce7641a0ffce, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731514037256
2024-11-13T16:07:19,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c84d48b11cf143a49c4e384a0142d3e7 is 1080, key is row0135/info:/1731514039268/Put/seqid=0
2024-11-13T16:07:19,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741861_1037 (size=22238)
2024-11-13T16:07:19,308 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741861_1037 (size=22238)
2024-11-13T16:07:19,309 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c84d48b11cf143a49c4e384a0142d3e7
2024-11-13T16:07:19,317 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#78 average throughput is 74.91 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:19,317 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/18eb8f0568014af7a4263c0737311d33 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c84d48b11cf143a49c4e384a0142d3e7 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7
2024-11-13T16:07:19,321 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7, entries=16, sequenceid=195, filesize=21.7 K
2024-11-13T16:07:19,322 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=10.51 KB/10760 for 9d56dfa1976de842f0efb20587b93a2b in 30ms, sequenceid=195, compaction requested=false
2024-11-13T16:07:19,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:19,322 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741862_1038 (size=84201)
2024-11-13T16:07:19,323 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741862_1038 (size=84201)
2024-11-13T16:07:19,327 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/18eb8f0568014af7a4263c0737311d33 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/18eb8f0568014af7a4263c0737311d33
2024-11-13T16:07:19,333 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into 18eb8f0568014af7a4263c0737311d33(size=82.2 K), total size for store is 103.9 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:19,333 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:19,333 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514039291; duration=0sec
2024-11-13T16:07:19,333 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:19,333 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:19,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:19,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:20,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:20,591 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:21,310 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-13T16:07:21,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/b89c41070ef34990b8fe13cea80a699a is 1080, key is row0151/info:/1731514039293/Put/seqid=0
2024-11-13T16:07:21,318 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741863_1039 (size=16828)
2024-11-13T16:07:21,319 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741863_1039 (size=16828)
2024-11-13T16:07:21,319 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/b89c41070ef34990b8fe13cea80a699a
2024-11-13T16:07:21,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/b89c41070ef34990b8fe13cea80a699a as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a
2024-11-13T16:07:21,329 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a, entries=11, sequenceid=210, filesize=16.4 K
2024-11-13T16:07:21,330 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for 9d56dfa1976de842f0efb20587b93a2b in 20ms, sequenceid=210, compaction requested=true
2024-11-13T16:07:21,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:21,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:21,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:21,330 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:21,331 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-13T16:07:21,332 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 123267 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:21,332 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:21,332 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:21,332 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/18eb8f0568014af7a4263c0737311d33, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=120.4 K
2024-11-13T16:07:21,332 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18eb8f0568014af7a4263c0737311d33, keycount=73, bloomtype=ROW, size=82.2 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731514013083
2024-11-13T16:07:21,333 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting c84d48b11cf143a49c4e384a0142d3e7, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731514039268
2024-11-13T16:07:21,333 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting b89c41070ef34990b8fe13cea80a699a, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731514039293
2024-11-13T16:07:21,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ded7c509667b4282b2c6f00b3577f560 is 1080, key is row0162/info:/1731514041311/Put/seqid=0
2024-11-13T16:07:21,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741864_1040 (size=20078)
2024-11-13T16:07:21,340 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741864_1040 (size=20078)
2024-11-13T16:07:21,340 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ded7c509667b4282b2c6f00b3577f560
2024-11-13T16:07:21,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ded7c509667b4282b2c6f00b3577f560 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560
2024-11-13T16:07:21,351 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#81 average throughput is 102.62 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:21,352 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c5e6ce4d31894c87997391d0ef4d5d40 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:21,353 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560, entries=14, sequenceid=227, filesize=19.6 K
2024-11-13T16:07:21,354 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for 9d56dfa1976de842f0efb20587b93a2b in 23ms, sequenceid=227, compaction requested=false
2024-11-13T16:07:21,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:21,356 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB
2024-11-13T16:07:21,358 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741865_1041 (size=113417)
2024-11-13T16:07:21,359 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741865_1041 (size=113417)
2024-11-13T16:07:21,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/a97f6bd9d8974010b8a57077c3899eee is 1080, key is row0176/info:/1731514041332/Put/seqid=0
2024-11-13T16:07:21,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741866_1042 (size=21156)
2024-11-13T16:07:21,364 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741866_1042 (size=21156)
2024-11-13T16:07:21,365 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/a97f6bd9d8974010b8a57077c3899eee
2024-11-13T16:07:21,365 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c5e6ce4d31894c87997391d0ef4d5d40 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c5e6ce4d31894c87997391d0ef4d5d40
2024-11-13T16:07:21,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/a97f6bd9d8974010b8a57077c3899eee as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee
2024-11-13T16:07:21,369 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into c5e6ce4d31894c87997391d0ef4d5d40(size=110.8 K), total size for store is 130.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:21,370 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:21,370 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514041330; duration=0sec
2024-11-13T16:07:21,370 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:21,370 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:21,373 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee, entries=15, sequenceid=245, filesize=20.7 K
2024-11-13T16:07:21,373 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=2.10 KB/2152 for 9d56dfa1976de842f0efb20587b93a2b in 17ms, sequenceid=245, compaction requested=true
2024-11-13T16:07:21,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:21,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:21,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:21,373 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:21,374 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 154651 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:21,374 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:21,374 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:21,375 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c5e6ce4d31894c87997391d0ef4d5d40, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=151.0 K
2024-11-13T16:07:21,375 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e6ce4d31894c87997391d0ef4d5d40, keycount=100, bloomtype=ROW, size=110.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731514013083
2024-11-13T16:07:21,375 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting ded7c509667b4282b2c6f00b3577f560, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731514041311
2024-11-13T16:07:21,375 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97f6bd9d8974010b8a57077c3899eee, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731514041332
2024-11-13T16:07:21,384 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#83 average throughput is 132.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:21,384 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d187fe2a1ce14426b5aea7b5a05d668b is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:21,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741867_1043 (size=144986)
2024-11-13T16:07:21,387 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741867_1043 (size=144986)
2024-11-13T16:07:21,391 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d187fe2a1ce14426b5aea7b5a05d668b as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d187fe2a1ce14426b5aea7b5a05d668b
2024-11-13T16:07:21,396 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into d187fe2a1ce14426b5aea7b5a05d668b(size=141.6 K), total size for store is 141.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:21,396 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:21,396 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514041373; duration=0sec
2024-11-13T16:07:21,396 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:21,396 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:21,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:21,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:22,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:22,592 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:23,367 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:07:23,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/496f35c3f8984422a5fedb31ded0c1aa is 1080, key is row0191/info:/1731514041356/Put/seqid=0
2024-11-13T16:07:23,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741868_1044 (size=12517)
2024-11-13T16:07:23,376 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741868_1044 (size=12517)
2024-11-13T16:07:23,377 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/496f35c3f8984422a5fedb31ded0c1aa
2024-11-13T16:07:23,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/496f35c3f8984422a5fedb31ded0c1aa as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa
2024-11-13T16:07:23,387 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa, entries=7, sequenceid=257, filesize=12.2 K
2024-11-13T16:07:23,388 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 9d56dfa1976de842f0efb20587b93a2b in 20ms, sequenceid=257, compaction requested=false
2024-11-13T16:07:23,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:23,389 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-13T16:07:23,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ff505de7c9684d1a9f12f25a94d7933c is 1080, key is row0198/info:/1731514043368/Put/seqid=0
2024-11-13T16:07:23,397 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741869_1045 (size=20092)
2024-11-13T16:07:23,398 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741869_1045 (size=20092)
2024-11-13T16:07:23,398 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ff505de7c9684d1a9f12f25a94d7933c
2024-11-13T16:07:23,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ff505de7c9684d1a9f12f25a94d7933c as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c
2024-11-13T16:07:23,408 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c, entries=14, sequenceid=274, filesize=19.6 K
2024-11-13T16:07:23,409 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 9d56dfa1976de842f0efb20587b93a2b in 19ms, sequenceid=274, compaction requested=true
2024-11-13T16:07:23,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:23,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:23,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:23,409 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:23,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:23,409 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB
2024-11-13T16:07:23,410 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 177595 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:23,410 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:23,410 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:23,410 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d187fe2a1ce14426b5aea7b5a05d668b, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=173.4 K
2024-11-13T16:07:23,410 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting d187fe2a1ce14426b5aea7b5a05d668b, keycount=129, bloomtype=ROW, size=141.6 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731514013083
2024-11-13T16:07:23,411 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting 496f35c3f8984422a5fedb31ded0c1aa, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731514041356
2024-11-13T16:07:23,411 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff505de7c9684d1a9f12f25a94d7933c, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731514043368
2024-11-13T16:07:23,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/e8e869a9a99a4870b8df2916799ae463 is 1080, key is row0212/info:/1731514043390/Put/seqid=0
2024-11-13T16:07:23,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741870_1046 (size=19013)
2024-11-13T16:07:23,417 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741870_1046 (size=19013)
2024-11-13T16:07:23,418 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/e8e869a9a99a4870b8df2916799ae463
2024-11-13T16:07:23,422 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#87 average throughput is 76.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:23,423 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/bad6e485895e48d3bceee787422985b1 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:23,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/e8e869a9a99a4870b8df2916799ae463 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463
2024-11-13T16:07:23,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741871_1047 (size=167745)
2024-11-13T16:07:23,426 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741871_1047 (size=167745)
2024-11-13T16:07:23,428 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463, entries=13, sequenceid=290, filesize=18.6 K
2024-11-13T16:07:23,429 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=0 B/0 for 9d56dfa1976de842f0efb20587b93a2b in 20ms, sequenceid=290, compaction requested=false
2024-11-13T16:07:23,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:23,431 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/bad6e485895e48d3bceee787422985b1 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/bad6e485895e48d3bceee787422985b1
2024-11-13T16:07:23,436 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into bad6e485895e48d3bceee787422985b1(size=163.8 K), total size for store is 182.4 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:23,436 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:23,436 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514043409; duration=0sec
2024-11-13T16:07:23,436 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:23,436 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:23,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:23,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:24,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:24,593 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:25,419 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB
2024-11-13T16:07:25,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d0234851d9654da588acba4d1fbd9bd9 is 1080, key is row0225/info:/1731514045411/Put/seqid=0
2024-11-13T16:07:25,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741872_1048 (size=12523)
2024-11-13T16:07:25,428 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741872_1048 (size=12523)
2024-11-13T16:07:25,429 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d0234851d9654da588acba4d1fbd9bd9
2024-11-13T16:07:25,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/d0234851d9654da588acba4d1fbd9bd9 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9
2024-11-13T16:07:25,439 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9, entries=7, sequenceid=301, filesize=12.2 K
2024-11-13T16:07:25,440 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 9d56dfa1976de842f0efb20587b93a2b in 21ms, sequenceid=301, compaction requested=true
2024-11-13T16:07:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9d56dfa1976de842f0efb20587b93a2b:info, priority=-2147483648, current under compaction store size is 1
2024-11-13T16:07:25,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:25,440 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking
2024-11-13T16:07:25,441 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3  files of size 199281 starting at candidate #0 after considering 1 permutations with 1 in ratio
2024-11-13T16:07:25,441 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1541): 9d56dfa1976de842f0efb20587b93a2b/info is initiating minor compaction (all files)
2024-11-13T16:07:25,441 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9d56dfa1976de842f0efb20587b93a2b/info in TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:25,441 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/bad6e485895e48d3bceee787422985b1, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9] into tmpdir=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp, totalSize=194.6 K
2024-11-13T16:07:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44287 {}] regionserver.HRegion(8855): Flush requested on 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:25,442 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=14.71 KB heapSize=16 KB
2024-11-13T16:07:25,442 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad6e485895e48d3bceee787422985b1, keycount=150, bloomtype=ROW, size=163.8 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731514013083
2024-11-13T16:07:25,442 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e869a9a99a4870b8df2916799ae463, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731514043390
2024-11-13T16:07:25,443 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0234851d9654da588acba4d1fbd9bd9, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731514045411
2024-11-13T16:07:25,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c0b94a8387354b449410196f580129ea is 1080, key is row0232/info:/1731514045420/Put/seqid=0
2024-11-13T16:07:25,450 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741873_1049 (size=20092)
2024-11-13T16:07:25,451 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741873_1049 (size=20092)
2024-11-13T16:07:25,451 INFO  [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c0b94a8387354b449410196f580129ea
2024-11-13T16:07:25,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/c0b94a8387354b449410196f580129ea as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c0b94a8387354b449410196f580129ea
2024-11-13T16:07:25,458 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9d56dfa1976de842f0efb20587b93a2b#info#compaction#90 average throughput is 58.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second
2024-11-13T16:07:25,459 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ba03d37fe8d544f09253a2ef5888a624 is 1080, key is row0062/info:/1731514013083/Put/seqid=0
2024-11-13T16:07:25,461 INFO  [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c0b94a8387354b449410196f580129ea, entries=14, sequenceid=318, filesize=19.6 K
2024-11-13T16:07:25,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741874_1050 (size=189435)
2024-11-13T16:07:25,461 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741874_1050 (size=189435)
2024-11-13T16:07:25,462 INFO  [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=11.56 KB/11836 for 9d56dfa1976de842f0efb20587b93a2b in 21ms, sequenceid=318, compaction requested=false
2024-11-13T16:07:25,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:25,466 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/ba03d37fe8d544f09253a2ef5888a624 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ba03d37fe8d544f09253a2ef5888a624
2024-11-13T16:07:25,471 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9d56dfa1976de842f0efb20587b93a2b/info of 9d56dfa1976de842f0efb20587b93a2b into ba03d37fe8d544f09253a2ef5888a624(size=185.0 K), total size for store is 204.6 K. This selection was in queue for 0sec, and took 0sec to execute.
2024-11-13T16:07:25,471 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:25,471 INFO  [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., storeName=9d56dfa1976de842f0efb20587b93a2b/info, priority=13, startTime=1731514045440; duration=0sec
2024-11-13T16:07:25,471 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0
2024-11-13T16:07:25,471 DEBUG [RS:0;c4c650e80f58:44287-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9d56dfa1976de842f0efb20587b93a2b:info
2024-11-13T16:07:25,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:25,594 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:26,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:26,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:27,458 INFO  [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files
2024-11-13T16:07:27,459 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44287%2C1731513989865.1731514047458
2024-11-13T16:07:27,465 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,465 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,465 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,465 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,465 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,465 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731513990248 with entries=311, filesize=307.82 KB; new WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731514047458
2024-11-13T16:07:27,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741833_1009 (size=315213)
2024-11-13T16:07:27,468 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741833_1009 (size=315213)
2024-11-13T16:07:27,468 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40329:40329),(127.0.0.1/127.0.0.1:42183:42183)]
2024-11-13T16:07:27,469 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731513990248 is not closed yet, will try archiving it next time
2024-11-13T16:07:27,473 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d56dfa1976de842f0efb20587b93a2b 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB
2024-11-13T16:07:27,478 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/88e16e9e8325471e84ee07d36410f8c0 is 1080, key is row0246/info:/1731514045442/Put/seqid=0
2024-11-13T16:07:27,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741876_1052 (size=16839)
2024-11-13T16:07:27,483 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741876_1052 (size=16839)
2024-11-13T16:07:27,483 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/88e16e9e8325471e84ee07d36410f8c0
2024-11-13T16:07:27,489 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/.tmp/info/88e16e9e8325471e84ee07d36410f8c0 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/88e16e9e8325471e84ee07d36410f8c0
2024-11-13T16:07:27,494 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/88e16e9e8325471e84ee07d36410f8c0, entries=11, sequenceid=333, filesize=16.4 K
2024-11-13T16:07:27,495 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 9d56dfa1976de842f0efb20587b93a2b in 22ms, sequenceid=333, compaction requested=true
2024-11-13T16:07:27,495 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9d56dfa1976de842f0efb20587b93a2b:

2024-11-13T16:07:27,495 INFO  [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB
2024-11-13T16:07:27,500 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/eeecb14855ea45e68d17cc470d93eda8 is 186, key is TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8./info:regioninfo/1731514013793/Put/seqid=0
2024-11-13T16:07:27,504 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741877_1053 (size=6153)
2024-11-13T16:07:27,505 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741877_1053 (size=6153)
2024-11-13T16:07:27,505 INFO  [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/eeecb14855ea45e68d17cc470d93eda8
2024-11-13T16:07:27,511 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/.tmp/info/eeecb14855ea45e68d17cc470d93eda8 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/info/eeecb14855ea45e68d17cc470d93eda8
2024-11-13T16:07:27,515 INFO  [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/info/eeecb14855ea45e68d17cc470d93eda8, entries=5, sequenceid=21, filesize=6.0 K
2024-11-13T16:07:27,516 INFO  [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false
2024-11-13T16:07:27,516 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740:

2024-11-13T16:07:27,516 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e83fccdcd88e4fbfaa390db8e2bf57d8:

2024-11-13T16:07:27,517 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C44287%2C1731513989865.1731514047517
2024-11-13T16:07:27,521 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,521 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,521 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,521 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,521 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,521 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731514047458 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731514047517
2024-11-13T16:07:27,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42183:42183),(127.0.0.1/127.0.0.1:40329:40329)]
2024-11-13T16:07:27,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731514047458 is not closed yet, will try archiving it next time
2024-11-13T16:07:27,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741875_1051 (size=731)
2024-11-13T16:07:27,523 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731513990248 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs/c4c650e80f58%2C44287%2C1731513989865.1731513990248
2024-11-13T16:07:27,523 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741875_1051 (size=731)
2024-11-13T16:07:27,523 INFO  [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1])
2024-11-13T16:07:27,524 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/WALs/c4c650e80f58,44287,1731513989865/c4c650e80f58%2C44287%2C1731513989865.1731514047458 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs/c4c650e80f58%2C44287%2C1731513989865.1731514047458
2024-11-13T16:07:27,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:27,595 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:27,623 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:07:27,624 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:07:27,624 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:27,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:27,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:27,624 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:07:27,624 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:07:27,624 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1914419557, stopped=false
2024-11-13T16:07:27,624 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,41807,1731513989821
2024-11-13T16:07:27,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:27,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:27,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:27,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:27,626 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:07:27,626 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:07:27,626 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:27,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:27,627 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,44287,1731513989865' *****
2024-11-13T16:07:27,627 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:07:27,627 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:27,627 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:27,627 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:07:27,627 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:07:27,627 INFO  [RS:0;c4c650e80f58:44287 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d56dfa1976de842f0efb20587b93a2b
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(3091): Received CLOSE for e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,44287,1731513989865
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:44287.
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d56dfa1976de842f0efb20587b93a2b, disabling compactions & flushes
2024-11-13T16:07:27,628 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:27,628 DEBUG [RS:0;c4c650e80f58:44287 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:27,628 DEBUG [RS:0;c4c650e80f58:44287 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b. after waiting 0 ms
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:07:27,628 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close
2024-11-13T16:07:27,628 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1325): Online Regions={9d56dfa1976de842f0efb20587b93a2b=TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b., 1588230740=hbase:meta,,1.1588230740, e83fccdcd88e4fbfaa390db8e2bf57d8=TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.}
2024-11-13T16:07:27,628 DEBUG [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9d56dfa1976de842f0efb20587b93a2b, e83fccdcd88e4fbfaa390db8e2bf57d8
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:07:27,628 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:07:27,628 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:07:27,629 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:07:27,628 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-top, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/42ec8b402afd4a1680151e23f5a212b7, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/64ddf4b4b74c4926aab26cd368e5f82a, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c3885e57a98147d18be4e11ee3474022, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/18eb8f0568014af7a4263c0737311d33, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c5e6ce4d31894c87997391d0ef4d5d40, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d187fe2a1ce14426b5aea7b5a05d668b, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/bad6e485895e48d3bceee787422985b1, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9] to archive
2024-11-13T16:07:27,630 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:07:27,631 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:07:27,633 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/42ec8b402afd4a1680151e23f5a212b7 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/42ec8b402afd4a1680151e23f5a212b7
2024-11-13T16:07:27,633 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1
2024-11-13T16:07:27,633 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:07:27,633 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:07:27,633 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731514047628Running coprocessor pre-close hooks at 1731514047628Disabling compacts and flushes for region at 1731514047628Disabling writes for close at 1731514047628Writing region close event to WAL at 1731514047630 (+2 ms)Running coprocessor post-close hooks at 1731514047633 (+3 ms)Closed at 1731514047633
2024-11-13T16:07:27,633 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:07:27,634 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/08a5511b76404359bdf45e697b39e14d
2024-11-13T16:07:27,635 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/64ddf4b4b74c4926aab26cd368e5f82a to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/64ddf4b4b74c4926aab26cd368e5f82a
2024-11-13T16:07:27,636 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ddf29ade2fe94133a92dd79a46dd529b
2024-11-13T16:07:27,637 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/62f914f2a2c5452bb736320143e74197
2024-11-13T16:07:27,638 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c3885e57a98147d18be4e11ee3474022 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c3885e57a98147d18be4e11ee3474022
2024-11-13T16:07:27,639 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d8b8aa1f0121452e966abf888424147d
2024-11-13T16:07:27,640 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/0556e5ab58a54dc6ab08c24e50063a68
2024-11-13T16:07:27,641 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/18eb8f0568014af7a4263c0737311d33 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/18eb8f0568014af7a4263c0737311d33
2024-11-13T16:07:27,642 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/1749449e03af4bc0855bce7641a0ffce
2024-11-13T16:07:27,643 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c84d48b11cf143a49c4e384a0142d3e7
2024-11-13T16:07:27,644 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c5e6ce4d31894c87997391d0ef4d5d40 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/c5e6ce4d31894c87997391d0ef4d5d40
2024-11-13T16:07:27,645 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/b89c41070ef34990b8fe13cea80a699a
2024-11-13T16:07:27,646 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ded7c509667b4282b2c6f00b3577f560
2024-11-13T16:07:27,646 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d187fe2a1ce14426b5aea7b5a05d668b to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d187fe2a1ce14426b5aea7b5a05d668b
2024-11-13T16:07:27,647 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/a97f6bd9d8974010b8a57077c3899eee
2024-11-13T16:07:27,648 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/496f35c3f8984422a5fedb31ded0c1aa
2024-11-13T16:07:27,649 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/bad6e485895e48d3bceee787422985b1 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/bad6e485895e48d3bceee787422985b1
2024-11-13T16:07:27,651 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/ff505de7c9684d1a9f12f25a94d7933c
2024-11-13T16:07:27,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/e8e869a9a99a4870b8df2916799ae463
2024-11-13T16:07:27,652 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9 to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/info/d0234851d9654da588acba4d1fbd9bd9
2024-11-13T16:07:27,653 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried.
org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c4c650e80f58:41807 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
	at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?]
	at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
	at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?]
	at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?]
	at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?]
	at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	... 16 more
2024-11-13T16:07:27,653 WARN  [StoreCloser-TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [42ec8b402afd4a1680151e23f5a212b7=8260, 08a5511b76404359bdf45e697b39e14d=12509, 64ddf4b4b74c4926aab26cd368e5f82a=33163, ddf29ade2fe94133a92dd79a46dd529b=22222, 62f914f2a2c5452bb736320143e74197=15741, c3885e57a98147d18be4e11ee3474022=61324, d8b8aa1f0121452e966abf888424147d=22238, 0556e5ab58a54dc6ab08c24e50063a68=20078, 18eb8f0568014af7a4263c0737311d33=84201, 1749449e03af4bc0855bce7641a0ffce=12516, c84d48b11cf143a49c4e384a0142d3e7=22238, c5e6ce4d31894c87997391d0ef4d5d40=113417, b89c41070ef34990b8fe13cea80a699a=16828, ded7c509667b4282b2c6f00b3577f560=20078, d187fe2a1ce14426b5aea7b5a05d668b=144986, a97f6bd9d8974010b8a57077c3899eee=21156, 496f35c3f8984422a5fedb31ded0c1aa=12517, bad6e485895e48d3bceee787422985b1=167745, ff505de7c9684d1a9f12f25a94d7933c=20092, e8e869a9a99a4870b8df2916799ae463=19013, d0234851d9654da588acba4d1fbd9bd9=12523]
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9d56dfa1976de842f0efb20587b93a2b/recovered.edits/336.seqid, newMaxSeqId=336, maxSeqId=85
2024-11-13T16:07:27,657 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d56dfa1976de842f0efb20587b93a2b:
Waiting for close lock at 1731514047628Running coprocessor pre-close hooks at 1731514047628Disabling compacts and flushes for region at 1731514047628Disabling writes for close at 1731514047628Writing region close event to WAL at 1731514047654 (+26 ms)Running coprocessor post-close hooks at 1731514047657 (+3 ms)Closed at 1731514047657
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731514013149.9d56dfa1976de842f0efb20587b93a2b.
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e83fccdcd88e4fbfaa390db8e2bf57d8, disabling compactions & flushes
2024-11-13T16:07:27,657 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8. after waiting 0 ms
2024-11-13T16:07:27,657 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:07:27,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d->hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/9cf1f5faf3b84cb9bdad9af6f5b3801d/info/a01a6a23f9174cb9a00e068d3593e086-bottom] to archive
2024-11-13T16:07:27,658 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.-1 {}] backup.HFileArchiver(360): Archiving compacted files.
2024-11-13T16:07:27,660 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d to hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/archive/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/info/a01a6a23f9174cb9a00e068d3593e086.9cf1f5faf3b84cb9bdad9af6f5b3801d
2024-11-13T16:07:27,660 WARN  [StoreCloser-TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.-1 {}] regionserver.HStore(2414): Failed to report archival of files: []
2024-11-13T16:07:27,663 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/data/default/TestLogRolling-testLogRolling/e83fccdcd88e4fbfaa390db8e2bf57d8/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=85
2024-11-13T16:07:27,663 INFO  [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:07:27,663 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e83fccdcd88e4fbfaa390db8e2bf57d8:
Waiting for close lock at 1731514047657Running coprocessor pre-close hooks at 1731514047657Disabling compacts and flushes for region at 1731514047657Disabling writes for close at 1731514047657Writing region close event to WAL at 1731514047660 (+3 ms)Running coprocessor post-close hooks at 1731514047663 (+3 ms)Closed at 1731514047663
2024-11-13T16:07:27,663 DEBUG [RS_CLOSE_REGION-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731514013149.e83fccdcd88e4fbfaa390db8e2bf57d8.
2024-11-13T16:07:27,828 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,44287,1731513989865; all regions closed.
2024-11-13T16:07:27,829 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,829 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,829 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,829 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,829 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,831 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741834_1010 (size=8107)
2024-11-13T16:07:27,832 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741834_1010 (size=8107)
2024-11-13T16:07:27,834 DEBUG [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs
2024-11-13T16:07:27,834 INFO  [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C44287%2C1731513989865.meta:.meta(num 1731513990643)
2024-11-13T16:07:27,834 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,834 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,834 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,834 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,834 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741878_1054 (size=778)
2024-11-13T16:07:27,836 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741878_1054 (size=778)
2024-11-13T16:07:27,838 DEBUG [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/oldWALs
2024-11-13T16:07:27,838 INFO  [RS:0;c4c650e80f58:44287 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C44287%2C1731513989865:(num 1731514047517)
2024-11-13T16:07:27,838 DEBUG [RS:0;c4c650e80f58:44287 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:27,838 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:07:27,838 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:07:27,838 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown
2024-11-13T16:07:27,838 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:07:27,838 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:07:27,839 INFO  [RS:0;c4c650e80f58:44287 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44287
2024-11-13T16:07:27,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,44287,1731513989865
2024-11-13T16:07:27,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:07:27,841 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:07:27,842 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,44287,1731513989865]
2024-11-13T16:07:27,844 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,44287,1731513989865 already deleted, retry=false
2024-11-13T16:07:27,844 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,44287,1731513989865 expired; onlineServers=0
2024-11-13T16:07:27,844 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,41807,1731513989821' *****
2024-11-13T16:07:27,844 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:07:27,845 DEBUG [M:0;c4c650e80f58:41807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:07:27,845 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:07:27,845 DEBUG [M:0;c4c650e80f58:41807 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:07:27,845 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513990041 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731513990041,5,FailOnTimeoutGroup]
2024-11-13T16:07:27,845 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513990042 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731513990042,5,FailOnTimeoutGroup]
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:07:27,845 DEBUG [M:0;c4c650e80f58:41807 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:07:27,845 INFO  [M:0;c4c650e80f58:41807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:07:27,845 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:07:27,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:07:27,846 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:27,846 DEBUG [M:0;c4c650e80f58:41807 {}] zookeeper.ZKUtil(347): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:07:27,846 WARN  [M:0;c4c650e80f58:41807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:07:27,847 INFO  [M:0;c4c650e80f58:41807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/.lastflushedseqids
2024-11-13T16:07:27,851 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741879_1055 (size=228)
2024-11-13T16:07:27,852 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741879_1055 (size=228)
2024-11-13T16:07:27,852 INFO  [M:0;c4c650e80f58:41807 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:07:27,852 INFO  [M:0;c4c650e80f58:41807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:07:27,852 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:07:27,852 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:27,852 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:27,852 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:07:27,852 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:27,853 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.41 KB heapSize=63.33 KB
2024-11-13T16:07:27,868 DEBUG [M:0;c4c650e80f58:41807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bdb0aed27b14955b3a6c965338a63df is 82, key is hbase:meta,,1/info:regioninfo/1731513990664/Put/seqid=0
2024-11-13T16:07:27,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741880_1056 (size=5672)
2024-11-13T16:07:27,873 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741880_1056 (size=5672)
2024-11-13T16:07:27,874 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bdb0aed27b14955b3a6c965338a63df
2024-11-13T16:07:27,899 DEBUG [M:0;c4c650e80f58:41807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cbd6e1d268b445da1f27120adefed97 is 749, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731513991160/Put/seqid=0
2024-11-13T16:07:27,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741881_1057 (size=7089)
2024-11-13T16:07:27,904 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741881_1057 (size=7089)
2024-11-13T16:07:27,905 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.80 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cbd6e1d268b445da1f27120adefed97
2024-11-13T16:07:27,909 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cbd6e1d268b445da1f27120adefed97
2024-11-13T16:07:27,923 DEBUG [M:0;c4c650e80f58:41807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7fa59c946cd4a37a96ca07ea613b789 is 69, key is c4c650e80f58,44287,1731513989865/rs:state/1731513990103/Put/seqid=0
2024-11-13T16:07:27,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741882_1058 (size=5156)
2024-11-13T16:07:27,927 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741882_1058 (size=5156)
2024-11-13T16:07:27,927 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7fa59c946cd4a37a96ca07ea613b789
2024-11-13T16:07:27,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:27,943 INFO  [RS:0;c4c650e80f58:44287 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:07:27,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44287-0x100a60a4ad00001, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:27,943 INFO  [RS:0;c4c650e80f58:44287 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,44287,1731513989865; zookeeper connection closed.
2024-11-13T16:07:27,943 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fa28613 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fa28613
2024-11-13T16:07:27,943 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:07:27,945 DEBUG [M:0;c4c650e80f58:41807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/30c60b38574e4c6e975e8b6d98d6fe7c is 52, key is load_balancer_on/state:d/1731513990791/Put/seqid=0
2024-11-13T16:07:27,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741883_1059 (size=5056)
2024-11-13T16:07:27,949 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741883_1059 (size=5056)
2024-11-13T16:07:27,950 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/30c60b38574e4c6e975e8b6d98d6fe7c
2024-11-13T16:07:27,954 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bdb0aed27b14955b3a6c965338a63df as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2bdb0aed27b14955b3a6c965338a63df
2024-11-13T16:07:27,958 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2bdb0aed27b14955b3a6c965338a63df, entries=8, sequenceid=125, filesize=5.5 K
2024-11-13T16:07:27,959 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6cbd6e1d268b445da1f27120adefed97 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cbd6e1d268b445da1f27120adefed97
2024-11-13T16:07:27,963 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6cbd6e1d268b445da1f27120adefed97
2024-11-13T16:07:27,963 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6cbd6e1d268b445da1f27120adefed97, entries=13, sequenceid=125, filesize=6.9 K
2024-11-13T16:07:27,964 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b7fa59c946cd4a37a96ca07ea613b789 as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7fa59c946cd4a37a96ca07ea613b789
2024-11-13T16:07:27,967 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b7fa59c946cd4a37a96ca07ea613b789, entries=1, sequenceid=125, filesize=5.0 K
2024-11-13T16:07:27,968 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/30c60b38574e4c6e975e8b6d98d6fe7c as hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/30c60b38574e4c6e975e8b6d98d6fe7c
2024-11-13T16:07:27,972 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34901/user/jenkins/test-data/418cd0ae-523a-1dd9-458f-38b4de76c32b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/30c60b38574e4c6e975e8b6d98d6fe7c, entries=1, sequenceid=125, filesize=4.9 K
2024-11-13T16:07:27,973 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false
2024-11-13T16:07:27,977 INFO  [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:27,977 DEBUG [M:0;c4c650e80f58:41807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731514047852Disabling compacts and flushes for region at 1731514047852Disabling writes for close at 1731514047852Obtaining lock to block concurrent updates at 1731514047853 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731514047853Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52639, getHeapSize=64784, getOffHeapSize=0, getCellsCount=148 at 1731514047853Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731514047853Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731514047853Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731514047868 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731514047868Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731514047878 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731514047899 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731514047899Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731514047909 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731514047922 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731514047922Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731514047931 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731514047945 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731514047945Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@187f73a1: reopening flushed file at 1731514047954 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cb36fce: reopening flushed file at 1731514047958 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577482f7: reopening flushed file at 1731514047963 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@157cd66a: reopening flushed file at 1731514047968 (+5 ms)Finished flush of dataSize ~51.41 KB/52639, heapSize ~63.27 KB/64784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false at 1731514047973 (+5 ms)Writing region close event to WAL at 1731514047977 (+4 ms)Closed at 1731514047977
2024-11-13T16:07:27,978 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,978 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,978 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,978 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,978 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:27,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38629 is added to blk_1073741830_1006 (size=61308)
2024-11-13T16:07:27,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36533 is added to blk_1073741830_1006 (size=61308)
2024-11-13T16:07:27,981 INFO  [M:0;c4c650e80f58:41807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:07:27,981 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:07:27,981 INFO  [M:0;c4c650e80f58:41807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41807
2024-11-13T16:07:27,981 INFO  [M:0;c4c650e80f58:41807 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:07:28,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:28,083 INFO  [M:0;c4c650e80f58:41807 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:07:28,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41807-0x100a60a4ad00000, quorum=127.0.0.1:63114, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:28,086 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4707bb9a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:28,086 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@286b8c80{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:28,086 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:28,086 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41c54b7e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:28,086 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5773e0ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:28,089 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:07:28,089 WARN  [BP-1342233215-172.17.0.3-1731513989155 heartbeating to localhost/127.0.0.1:34901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:07:28,090 WARN  [BP-1342233215-172.17.0.3-1731513989155 heartbeating to localhost/127.0.0.1:34901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1342233215-172.17.0.3-1731513989155 (Datanode Uuid c7b1ab68-bac6-414b-ab8d-4bc06d47bdbb) service to localhost/127.0.0.1:34901
2024-11-13T16:07:28,090 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:07:28,090 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data3/current/BP-1342233215-172.17.0.3-1731513989155 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:28,090 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data4/current/BP-1342233215-172.17.0.3-1731513989155 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:28,091 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:07:28,092 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4063a419{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:28,093 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c0c72e{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:28,093 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:28,093 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417c1a7a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:28,093 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59fd7f75{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:28,094 WARN  [BP-1342233215-172.17.0.3-1731513989155 heartbeating to localhost/127.0.0.1:34901 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:07:28,094 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:07:28,094 WARN  [BP-1342233215-172.17.0.3-1731513989155 heartbeating to localhost/127.0.0.1:34901 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1342233215-172.17.0.3-1731513989155 (Datanode Uuid 96b784ac-5f7b-49a0-981c-64df61c81141) service to localhost/127.0.0.1:34901
2024-11-13T16:07:28,094 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:07:28,095 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data1/current/BP-1342233215-172.17.0.3-1731513989155 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:28,095 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/cluster_b4a166e1-6246-fd7b-fc7a-324153f11c23/data/data2/current/BP-1342233215-172.17.0.3-1731513989155 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:28,095 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:07:28,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51585bde{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:07:28,101 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15c8c411{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:28,101 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:28,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45bda0cb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:28,101 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a67ff9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:28,122 INFO  [regionserver/c4c650e80f58:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:07:28,128 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:07:28,159 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:07:28,168 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=229 (was 205)
Potentially hanging thread: HMaster-EventLoopGroup-14-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34901 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34901 from jenkins.hfs.6
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:34901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-41-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:34901
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-14-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34901
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-38-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-39-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-40-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34901 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: HMaster-EventLoopGroup-14-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=36 (was 69), ProcessCount=11 (was 11), AvailableMemoryMB=11262 (was 11366)
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=229, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=36, ProcessCount=11, AvailableMemoryMB=11262
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false}
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.log.dir so I do NOT create it in target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/fa83d0e2-83a3-beab-a3fc-61026d73c957/hadoop.tmp.dir so I do NOT create it in target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de, deleteOnExit=true
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS
2024-11-13T16:07:28,176 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/test.cache.data in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.tmp.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/mapreduce.cluster.local.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/mapreduce.cluster.temp.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF
2024-11-13T16:07:28,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.node-labels.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.nodemanager.log-dirs in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/nfs.dump.dir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/java.io.tmpdir in system properties and HBase conf
2024-11-13T16:07:28,177 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/dfs.journalnode.edits.dir in system properties and HBase conf
2024-11-13T16:07:28,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf
2024-11-13T16:07:28,178 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/fs.s3a.committer.staging.tmp.path in system properties and HBase conf
2024-11-13T16:07:28,191 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:07:28,253 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:07:28,257 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:07:28,258 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:07:28,258 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:07:28,258 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms
2024-11-13T16:07:28,259 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:07:28,259 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56836339{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:07:28,260 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@49428dc3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:07:28,374 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@377e4a58{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/java.io.tmpdir/jetty-localhost-44887-hadoop-hdfs-3_4_1-tests_jar-_-any-9215340138258310569/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:07:28,374 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8035060{HTTP/1.1, (http/1.1)}{localhost:44887}
2024-11-13T16:07:28,374 INFO  [Time-limited test {}] server.Server(415): Started @297260ms
2024-11-13T16:07:28,387 WARN  [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000.
2024-11-13T16:07:28,459 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:07:28,464 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:07:28,467 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:07:28,467 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:07:28,467 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:07:28,469 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e4376a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:07:28,469 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acdff1a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:07:28,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:28,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:28,600 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a35d82b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/java.io.tmpdir/jetty-localhost-45775-hadoop-hdfs-3_4_1-tests_jar-_-any-12788204120966810437/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:28,601 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@591676f4{HTTP/1.1, (http/1.1)}{localhost:45775}
2024-11-13T16:07:28,601 INFO  [Time-limited test {}] server.Server(415): Started @297487ms
2024-11-13T16:07:28,602 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:07:28,632 WARN  [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret
2024-11-13T16:07:28,634 INFO  [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9
2024-11-13T16:07:28,635 INFO  [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0
2024-11-13T16:07:28,635 INFO  [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults
2024-11-13T16:07:28,635 INFO  [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms
2024-11-13T16:07:28,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@698d258d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,AVAILABLE}
2024-11-13T16:07:28,635 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3829c39e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE}
2024-11-13T16:07:28,696 WARN  [Thread-2487 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data1/current/BP-523714403-172.17.0.3-1731514048197/current, will proceed with Du for space computation calculation, 
2024-11-13T16:07:28,696 WARN  [Thread-2488 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data2/current/BP-523714403-172.17.0.3-1731514048197/current, will proceed with Du for space computation calculation, 
2024-11-13T16:07:28,716 WARN  [Thread-2466 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:07:28,719 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41528877e6b21f6e with lease ID 0x53cd4e55bc658e72: Processing first storage report for DS-79a8de09-5215-4784-b7b9-104a09b71ab3 from datanode DatanodeRegistration(127.0.0.1:38925, datanodeUuid=87dc5a08-9c15-4ff6-ba75-3ece44e674e3, infoPort=36381, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197)
2024-11-13T16:07:28,719 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41528877e6b21f6e with lease ID 0x53cd4e55bc658e72: from storage DS-79a8de09-5215-4784-b7b9-104a09b71ab3 node DatanodeRegistration(127.0.0.1:38925, datanodeUuid=87dc5a08-9c15-4ff6-ba75-3ece44e674e3, infoPort=36381, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:07:28,719 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x41528877e6b21f6e with lease ID 0x53cd4e55bc658e72: Processing first storage report for DS-72f2258d-7cfc-4e55-9e6c-9c2ea944cb4c from datanode DatanodeRegistration(127.0.0.1:38925, datanodeUuid=87dc5a08-9c15-4ff6-ba75-3ece44e674e3, infoPort=36381, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197)
2024-11-13T16:07:28,719 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x41528877e6b21f6e with lease ID 0x53cd4e55bc658e72: from storage DS-72f2258d-7cfc-4e55-9e6c-9c2ea944cb4c node DatanodeRegistration(127.0.0.1:38925, datanodeUuid=87dc5a08-9c15-4ff6-ba75-3ece44e674e3, infoPort=36381, infoSecurePort=0, ipcPort=44933, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:07:28,754 INFO  [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5982a43f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/java.io.tmpdir/jetty-localhost-43487-hadoop-hdfs-3_4_1-tests_jar-_-any-8684818021776762398/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:28,754 INFO  [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@734134af{HTTP/1.1, (http/1.1)}{localhost:43487}
2024-11-13T16:07:28,754 INFO  [Time-limited test {}] server.Server(415): Started @297640ms
2024-11-13T16:07:28,755 WARN  [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering.
2024-11-13T16:07:28,847 WARN  [Thread-2514 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data4/current/BP-523714403-172.17.0.3-1731514048197/current, will proceed with Du for space computation calculation, 
2024-11-13T16:07:28,847 WARN  [Thread-2513 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data3/current/BP-523714403-172.17.0.3-1731514048197/current, will proceed with Du for space computation calculation, 
2024-11-13T16:07:28,864 WARN  [Thread-2502 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1
2024-11-13T16:07:28,866 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4736dab96a4fad58 with lease ID 0x53cd4e55bc658e73: Processing first storage report for DS-89acd612-a043-46fc-8a72-b3543e392b47 from datanode DatanodeRegistration(127.0.0.1:36799, datanodeUuid=3994a46a-b80b-4d42-8cdd-c2a209448a38, infoPort=42441, infoSecurePort=0, ipcPort=42169, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197)
2024-11-13T16:07:28,866 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4736dab96a4fad58 with lease ID 0x53cd4e55bc658e73: from storage DS-89acd612-a043-46fc-8a72-b3543e392b47 node DatanodeRegistration(127.0.0.1:36799, datanodeUuid=3994a46a-b80b-4d42-8cdd-c2a209448a38, infoPort=42441, infoSecurePort=0, ipcPort=42169, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:07:28,866 INFO  [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4736dab96a4fad58 with lease ID 0x53cd4e55bc658e73: Processing first storage report for DS-7ecd2325-a442-4ef9-a384-44bec28533b2 from datanode DatanodeRegistration(127.0.0.1:36799, datanodeUuid=3994a46a-b80b-4d42-8cdd-c2a209448a38, infoPort=42441, infoSecurePort=0, ipcPort=42169, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197)
2024-11-13T16:07:28,866 INFO  [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4736dab96a4fad58 with lease ID 0x53cd4e55bc658e73: from storage DS-7ecd2325-a442-4ef9-a384-44bec28533b2 node DatanodeRegistration(127.0.0.1:36799, datanodeUuid=3994a46a-b80b-4d42-8cdd-c2a209448a38, infoPort=42441, infoSecurePort=0, ipcPort=42169, storageInfo=lv=-57;cid=testClusterID;nsid=901699407;c=1731514048197), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
2024-11-13T16:07:28,876 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957
2024-11-13T16:07:28,879 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/zookeeper_0, clientPort=57468, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/zookeeper_0/version-2, dataDirSize=457
dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/zookeeper_0/version-2, dataLogSize=457
tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0

2024-11-13T16:07:28,880 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57468
2024-11-13T16:07:28,881 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,882 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,889 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:07:28,890 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741825_1001 (size=7)
2024-11-13T16:07:28,890 INFO  [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430 with version=8
2024-11-13T16:07:28,891 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:38585/user/jenkins/test-data/60c0680b-5bcc-d38f-cdf3-aa844d14d424/hbase-staging
2024-11-13T16:07:28,892 INFO  [Time-limited test {}] client.ConnectionUtils(128): master/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:07:28,892 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,892 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,892 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:07:28,893 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,893 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:07:28,893 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService
2024-11-13T16:07:28,893 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:07:28,893 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38989
2024-11-13T16:07:28,894 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38989 connecting to ZooKeeper ensemble=127.0.0.1:57468
2024-11-13T16:07:28,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:389890x0, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:07:28,901 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38989-0x100a60b318e0000 connected
2024-11-13T16:07:28,921 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,922 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,924 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:28,924 INFO  [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430, hbase.cluster.distributed=false
2024-11-13T16:07:28,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:07:28,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38989
2024-11-13T16:07:28,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38989
2024-11-13T16:07:28,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38989
2024-11-13T16:07:28,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38989
2024-11-13T16:07:28,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38989
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] client.ConnectionUtils(128): regionserver/c4c650e80f58:0 server-side Connection retries=45
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService
2024-11-13T16:07:28,941 INFO  [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation
2024-11-13T16:07:28,942 INFO  [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46683
2024-11-13T16:07:28,943 INFO  [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46683 connecting to ZooKeeper ensemble=127.0.0.1:57468
2024-11-13T16:07:28,943 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,944 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466830x0, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null
2024-11-13T16:07:28,949 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46683-0x100a60b318e0001 connected
2024-11-13T16:07:28,949 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:28,949 INFO  [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB
2024-11-13T16:07:28,949 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5
2024-11-13T16:07:28,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master
2024-11-13T16:07:28,951 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl
2024-11-13T16:07:28,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46683
2024-11-13T16:07:28,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46683
2024-11-13T16:07:28,951 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46683
2024-11-13T16:07:28,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46683
2024-11-13T16:07:28,952 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46683
2024-11-13T16:07:28,963 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c4c650e80f58:38989
2024-11-13T16:07:28,963 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c4c650e80f58,38989,1731514048892
2024-11-13T16:07:28,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:07:28,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:07:28,965 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c4c650e80f58,38989,1731514048892
2024-11-13T16:07:28,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master
2024-11-13T16:07:28,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:28,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:28,969 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on existing znode=/hbase/master
2024-11-13T16:07:28,969 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c4c650e80f58,38989,1731514048892 from backup master directory
2024-11-13T16:07:28,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c4c650e80f58,38989,1731514048892
2024-11-13T16:07:28,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:07:28,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters
2024-11-13T16:07:28,971 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:07:28,971 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c4c650e80f58,38989,1731514048892
2024-11-13T16:07:28,974 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/hbase.id] with ID: aaa77606-e570-4b3c-bcff-614e7322dea0
2024-11-13T16:07:28,974 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/.tmp/hbase.id
2024-11-13T16:07:28,979 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:07:28,980 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741826_1002 (size=42)
2024-11-13T16:07:28,980 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/.tmp/hbase.id]:[hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/hbase.id]
2024-11-13T16:07:28,990 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:28,990 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem.
2024-11-13T16:07:28,991 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms.
2024-11-13T16:07:28,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:28,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:28,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:07:28,999 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741827_1003 (size=196)
2024-11-13T16:07:28,999 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}
2024-11-13T16:07:29,000 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000
2024-11-13T16:07:29,000 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:07:29,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:07:29,006 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741828_1004 (size=1189)
2024-11-13T16:07:29,007 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store
2024-11-13T16:07:29,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:07:29,011 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741829_1005 (size=34)
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:07:29,012 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:29,012 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:29,012 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731514049012Disabling compacts and flushes for region at 1731514049012Disabling writes for close at 1731514049012Writing region close event to WAL at 1731514049012Closed at 1731514049012
2024-11-13T16:07:29,013 WARN  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/.initializing
2024-11-13T16:07:29,013 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/WALs/c4c650e80f58,38989,1731514048892
2024-11-13T16:07:29,015 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C38989%2C1731514048892, suffix=, logDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/WALs/c4c650e80f58,38989,1731514048892, archiveDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/oldWALs, maxLogs=10
2024-11-13T16:07:29,015 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C38989%2C1731514048892.1731514049015
2024-11-13T16:07:29,019 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/WALs/c4c650e80f58,38989,1731514048892/c4c650e80f58%2C38989%2C1731514048892.1731514049015
2024-11-13T16:07:29,023 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36381:36381),(127.0.0.1/127.0.0.1:42441:42441)]
2024-11-13T16:07:29,023 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:07:29,023 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:07:29,023 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,023 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,025 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:07:29,026 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info
2024-11-13T16:07:29,026 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,026 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,026 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:07:29,027 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc
2024-11-13T16:07:29,027 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,028 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:07:29,028 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:07:29,029 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs
2024-11-13T16:07:29,029 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,029 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:07:29,029 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 
2024-11-13T16:07:29,030 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state
2024-11-13T16:07:29,030 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,030 INFO  [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE
2024-11-13T16:07:29,030 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,031 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,031 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,032 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,032 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,033 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead.
2024-11-13T16:07:29,034 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682
2024-11-13T16:07:29,035 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:07:29,036 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874145, jitterRate=0.11153295636177063}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432}
2024-11-13T16:07:29,036 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682:
Writing region info on filesystem at 1731514049024Initializing all the Stores at 1731514049024Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049024Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514049024Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514049024Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514049024Cleaning up temporary data from old regions at 1731514049032 (+8 ms)Region opened successfully at 1731514049036 (+4 ms)
2024-11-13T16:07:29,036 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4
2024-11-13T16:07:29,039 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373da6db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:07:29,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating...
2024-11-13T16:07:29,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5
2024-11-13T16:07:29,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50
2024-11-13T16:07:29,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery...
2024-11-13T16:07:29,040 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec
2024-11-13T16:07:29,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec
2024-11-13T16:07:29,041 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150
2024-11-13T16:07:29,044 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'.
2024-11-13T16:07:29,044 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error)
2024-11-13T16:07:29,046 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false
2024-11-13T16:07:29,046 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1
2024-11-13T16:07:29,046 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error)
2024-11-13T16:07:29,049 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false
2024-11-13T16:07:29,049 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited
2024-11-13T16:07:29,050 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error)
2024-11-13T16:07:29,051 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false
2024-11-13T16:07:29,052 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error)
2024-11-13T16:07:29,053 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false
2024-11-13T16:07:29,054 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error)
2024-11-13T16:07:29,056 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false
2024-11-13T16:07:29,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:29,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:29,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,059 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c4c650e80f58,38989,1731514048892, sessionid=0x100a60b318e0000, setting cluster-up flag (Was=false)
2024-11-13T16:07:29,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,068 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort
2024-11-13T16:07:29,069 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,38989,1731514048892
2024-11-13T16:07:29,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,077 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort
2024-11-13T16:07:29,078 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c4c650e80f58,38989,1731514048892
2024-11-13T16:07:29,079 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again
2024-11-13T16:07:29,080 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta
2024-11-13T16:07:29,080 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2
2024-11-13T16:07:29,080 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc.
2024-11-13T16:07:29,081 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c4c650e80f58,38989,1731514048892
Number of backup masters: 0
Number of live region servers: 0
Number of dead region servers: 0
Number of unknown region servers: 0
Average load: 0.0
Number of requests: 0
Number of regions: 0
Number of regions in transition: 0
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c4c650e80f58:0, corePoolSize=5, maxPoolSize=5
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c4c650e80f58:0, corePoolSize=10, maxPoolSize=10
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:07:29,082 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,082 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731514079082
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,083 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:07:29,083 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner
2024-11-13T16:07:29,083 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner
2024-11-13T16:07:29,084 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731514049084,5,FailOnTimeoutGroup]
2024-11-13T16:07:29,084 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731514049084,5,FailOnTimeoutGroup]
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it.
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,084 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,084 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,085 INFO  [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}
2024-11-13T16:07:29,089 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:07:29,090 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741831_1007 (size=1321)
2024-11-13T16:07:29,091 INFO  [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321
2024-11-13T16:07:29,091 INFO  [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430
2024-11-13T16:07:29,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:07:29,096 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741832_1008 (size=32)
2024-11-13T16:07:29,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:07:29,098 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:07:29,099 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:07:29,099 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,099 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,099 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:07:29,100 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:07:29,100 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,100 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,100 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:07:29,101 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:07:29,101 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,101 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,101 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:07:29,102 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:07:29,102 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,103 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:07:29,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740
2024-11-13T16:07:29,103 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740
2024-11-13T16:07:29,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:07:29,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:07:29,105 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:07:29,105 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:07:29,107 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1
2024-11-13T16:07:29,107 INFO  [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849953, jitterRate=0.08077125251293182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:07:29,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740:
Writing region info on filesystem at 1731514049097Initializing all the Stores at 1731514049097Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049097Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049097Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514049097Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049097Cleaning up temporary data from old regions at 1731514049104 (+7 ms)Region opened successfully at 1731514049108 (+4 ms)
2024-11-13T16:07:29,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:07:29,108 INFO  [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:07:29,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:07:29,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:07:29,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:07:29,109 INFO  [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:07:29,109 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731514049108Disabling compacts and flushes for region at 1731514049108Disabling writes for close at 1731514049108Writing region close event to WAL at 1731514049109 (+1 ms)Closed at 1731514049109
2024-11-13T16:07:29,110 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:07:29,110 INFO  [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta
2024-11-13T16:07:29,110 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}]
2024-11-13T16:07:29,111 INFO  [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:07:29,112 INFO  [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false
2024-11-13T16:07:29,154 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(746): ClusterId : aaa77606-e570-4b3c-bcff-614e7322dea0
2024-11-13T16:07:29,154 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing
2024-11-13T16:07:29,156 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized
2024-11-13T16:07:29,156 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing
2024-11-13T16:07:29,159 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized
2024-11-13T16:07:29,159 DEBUG [RS:0;c4c650e80f58:46683 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117a5da2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c4c650e80f58/172.17.0.3:0
2024-11-13T16:07:29,171 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c4c650e80f58:46683
2024-11-13T16:07:29,171 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled
2024-11-13T16:07:29,171 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled
2024-11-13T16:07:29,171 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(832): About to register with Master.
2024-11-13T16:07:29,172 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(2659): reportForDuty to master=c4c650e80f58,38989,1731514048892 with port=46683, startcode=1731514048941
2024-11-13T16:07:29,172 DEBUG [RS:0;c4c650e80f58:46683 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false
2024-11-13T16:07:29,174 INFO  [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58333, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService
2024-11-13T16:07:29,175 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38989 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,175 INFO  [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38989 {}] master.ServerManager(517): Registering regionserver=c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,176 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430
2024-11-13T16:07:29,176 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36573
2024-11-13T16:07:29,176 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1
2024-11-13T16:07:29,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:07:29,178 DEBUG [RS:0;c4c650e80f58:46683 {}] zookeeper.ZKUtil(111): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,178 WARN  [RS:0;c4c650e80f58:46683 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!)
2024-11-13T16:07:29,178 INFO  [RS:0;c4c650e80f58:46683 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:07:29,179 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,179 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c4c650e80f58,46683,1731514048941]
2024-11-13T16:07:29,182 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds
2024-11-13T16:07:29,183 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false
2024-11-13T16:07:29,183 INFO  [RS:0;c4c650e80f58:46683 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms
2024-11-13T16:07:29,183 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,184 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S
2024-11-13T16:07:29,184 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 
2024-11-13T16:07:29,184 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,184 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,184 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,184 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,184 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c4c650e80f58:0, corePoolSize=2, maxPoolSize=2
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c4c650e80f58:0, corePoolSize=1, maxPoolSize=1
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:07:29,185 DEBUG [RS:0;c4c650e80f58:46683 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c4c650e80f58:0, corePoolSize=3, maxPoolSize=3
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,185 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,46683,1731514048941-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:07:29,200 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false
2024-11-13T16:07:29,200 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,46683,1731514048941-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,200 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,200 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.Replication(171): c4c650e80f58,46683,1731514048941 started
2024-11-13T16:07:29,214 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,214 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1482): Serving as c4c650e80f58,46683,1731514048941, RpcServer on c4c650e80f58/172.17.0.3:46683, sessionid=0x100a60b318e0001
2024-11-13T16:07:29,214 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting
2024-11-13T16:07:29,214 DEBUG [RS:0;c4c650e80f58:46683 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,214 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,46683,1731514048941'
2024-11-13T16:07:29,214 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort'
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired'
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c4c650e80f58,46683,1731514048941'
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort'
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired'
2024-11-13T16:07:29,215 DEBUG [RS:0;c4c650e80f58:46683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started
2024-11-13T16:07:29,215 INFO  [RS:0;c4c650e80f58:46683 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled
2024-11-13T16:07:29,215 INFO  [RS:0;c4c650e80f58:46683 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager.
2024-11-13T16:07:29,262 WARN  [c4c650e80f58:38989 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions.
2024-11-13T16:07:29,317 INFO  [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C46683%2C1731514048941, suffix=, logDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/c4c650e80f58,46683,1731514048941, archiveDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs, maxLogs=32
2024-11-13T16:07:29,318 INFO  [RS:0;c4c650e80f58:46683 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C46683%2C1731514048941.1731514049317
2024-11-13T16:07:29,323 INFO  [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/c4c650e80f58,46683,1731514048941/c4c650e80f58%2C46683%2C1731514048941.1731514049317
2024-11-13T16:07:29,323 DEBUG [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42441:42441),(127.0.0.1/127.0.0.1:36381:36381)]
2024-11-13T16:07:29,512 DEBUG [c4c650e80f58:38989 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1
2024-11-13T16:07:29,513 INFO  [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,514 INFO  [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,46683,1731514048941, state=OPENING
2024-11-13T16:07:29,516 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it
2024-11-13T16:07:29,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,518 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN
2024-11-13T16:07:29,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:07:29,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:07:29,518 INFO  [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,46683,1731514048941}]
2024-11-13T16:07:29,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,34133,1731513857127/c4c650e80f58%2C34133%2C1731513857127.1731513857337
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:29,596 WARN  [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:46231/user/jenkins/test-data/f8111506-18a6-dddb-f9e9-1c7341e3ec6b/WALs/c4c650e80f58,41961,1731513856079/c4c650e80f58%2C41961%2C1731513856079.meta.1731513856932.meta
java.lang.reflect.InvocationTargetException: null
	at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?]
	at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
	at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?]
	at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?]
	at java.lang.Thread.run(Thread.java:840) ~[?:?]
Caused by: java.io.IOException: Filesystem closed
	at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?]
	at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?]
	at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?]
	... 11 more
2024-11-13T16:07:29,671 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false
2024-11-13T16:07:29,673 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33095, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService
2024-11-13T16:07:29,676 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740
2024-11-13T16:07:29,676 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:07:29,677 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c4c650e80f58%2C46683%2C1731514048941.meta, suffix=.meta, logDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/c4c650e80f58,46683,1731514048941, archiveDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs, maxLogs=32
2024-11-13T16:07:29,678 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c4c650e80f58%2C46683%2C1731514048941.meta.1731514049678.meta
2024-11-13T16:07:29,682 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/c4c650e80f58,46683,1731514048941/c4c650e80f58%2C46683%2C1731514048941.meta.1731514049678.meta
2024-11-13T16:07:29,686 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42441:42441),(127.0.0.1/127.0.0.1:36381:36381)]
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService
2024-11-13T16:07:29,691 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully.
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740
2024-11-13T16:07:29,691 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740
2024-11-13T16:07:29,693 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 
2024-11-13T16:07:29,694 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info
2024-11-13T16:07:29,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,694 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,694 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 
2024-11-13T16:07:29,695 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns
2024-11-13T16:07:29,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,695 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,695 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 
2024-11-13T16:07:29,696 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier
2024-11-13T16:07:29,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,696 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,696 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 
2024-11-13T16:07:29,697 INFO  [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table
2024-11-13T16:07:29,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker
2024-11-13T16:07:29,697 INFO  [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table,  memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE
2024-11-13T16:07:29,697 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740
2024-11-13T16:07:29,698 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740
2024-11-13T16:07:29,699 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740
2024-11-13T16:07:29,699 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740
2024-11-13T16:07:29,699 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740
2024-11-13T16:07:29,700 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead.
2024-11-13T16:07:29,701 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740
2024-11-13T16:07:29,701 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=715411, jitterRate=-0.09030893445014954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216}
2024-11-13T16:07:29,701 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740
2024-11-13T16:07:29,702 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740:
Running coprocessor pre-open hook at 1731514049691Writing region info on filesystem at 1731514049691Initializing all the Stores at 1731514049692 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049692Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049693 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731514049693Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731514049693Cleaning up temporary data from old regions at 1731514049699 (+6 ms)Running coprocessor post-open hooks at 1731514049701 (+2 ms)Region opened successfully at 1731514049702 (+1 ms)
2024-11-13T16:07:29,703 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731514049671
2024-11-13T16:07:29,705 DEBUG [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740
2024-11-13T16:07:29,705 INFO  [RS_OPEN_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740
2024-11-13T16:07:29,706 INFO  [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,706 INFO  [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c4c650e80f58,46683,1731514048941, state=OPEN
2024-11-13T16:07:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:07:29,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server
2024-11-13T16:07:29,713 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:07:29,713 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED
2024-11-13T16:07:29,715 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2
2024-11-13T16:07:29,715 INFO  [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c4c650e80f58,46683,1731514048941 in 195 msec
2024-11-13T16:07:29,716 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1
2024-11-13T16:07:29,716 INFO  [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec
2024-11-13T16:07:29,717 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta
2024-11-13T16:07:29,717 INFO  [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces
2024-11-13T16:07:29,718 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:07:29,718 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,46683,1731514048941, seqNum=-1]
2024-11-13T16:07:29,719 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:07:29,720 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:07:29,724 INFO  [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 643 msec
2024-11-13T16:07:29,724 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731514049724, completionTime=-1
2024-11-13T16:07:29,724 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running
2024-11-13T16:07:29,724 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster...
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731514109726
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731514169726
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-BalancerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c4c650e80f58:38989, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,726 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,727 DEBUG [master/c4c650e80f58:0.Chore.1 {}] janitor.CatalogJanitor(180): 
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.761sec
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting.
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting.
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting.
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-MobFileCleanerChore, period=86400, unit=SECONDS is enabled.
2024-11-13T16:07:29,732 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-MobFileCompactionChore, period=604800, unit=SECONDS is enabled.
2024-11-13T16:07:29,734 DEBUG [master/c4c650e80f58:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds
2024-11-13T16:07:29,734 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled.
2024-11-13T16:07:29,734 INFO  [master/c4c650e80f58:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c4c650e80f58,38989,1731514048892-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled.
2024-11-13T16:07:29,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f1960dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:07:29,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c4c650e80f58,38989,-1 for getting cluster id
2024-11-13T16:07:29,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false
2024-11-13T16:07:29,755 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'aaa77606-e570-4b3c-bcff-614e7322dea0'
2024-11-13T16:07:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse
2024-11-13T16:07:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "aaa77606-e570-4b3c-bcff-614e7322dea0"

2024-11-13T16:07:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff46454, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:07:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c4c650e80f58,38989,-1]
2024-11-13T16:07:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false
2024-11-13T16:07:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:29,757 INFO  [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService
2024-11-13T16:07:29,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698122cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null
2024-11-13T16:07:29,758 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry
2024-11-13T16:07:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c4c650e80f58,46683,1731514048941, seqNum=-1]
2024-11-13T16:07:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false
2024-11-13T16:07:29,760 INFO  [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService
2024-11-13T16:07:29,761 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c4c650e80f58,38989,1731514048892
2024-11-13T16:07:29,761 INFO  [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks
2024-11-13T16:07:29,764 INFO  [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false
2024-11-13T16:07:29,764 INFO  [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider
2024-11-13T16:07:29,765 INFO  [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/test.com,8080,1, archiveDir=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs, maxLogs=32
2024-11-13T16:07:29,766 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731514049766
2024-11-13T16:07:29,771 INFO  [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/test.com,8080,1/test.com%2C8080%2C1.1731514049766
2024-11-13T16:07:29,772 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42441:42441),(127.0.0.1/127.0.0.1:36381:36381)]
2024-11-13T16:07:29,773 INFO  [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731514049773
2024-11-13T16:07:29,777 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,777 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,777 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,778 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,778 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,778 INFO  [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/test.com,8080,1/test.com%2C8080%2C1.1731514049766 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/test.com,8080,1/test.com%2C8080%2C1.1731514049773
2024-11-13T16:07:29,779 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741835_1011 (size=93)
2024-11-13T16:07:29,780 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741835_1011 (size=93)
2024-11-13T16:07:29,781 INFO  [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/WALs/test.com,8080,1/test.com%2C8080%2C1.1731514049766 to hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs/test.com%2C8080%2C1.1731514049766
2024-11-13T16:07:29,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42441:42441),(127.0.0.1/127.0.0.1:36381:36381)]
2024-11-13T16:07:29,781 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,782 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,782 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,782 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,782 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741836_1012 (size=93)
2024-11-13T16:07:29,783 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741836_1012 (size=93)
2024-11-13T16:07:29,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs
2024-11-13T16:07:29,785 INFO  [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731514049773)
2024-11-13T16:07:29,785 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster
2024-11-13T16:07:29,785 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:07:29,786 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79)
    at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611)
    at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:29,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:29,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:29,786 INFO  [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited.
2024-11-13T16:07:29,786 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster
2024-11-13T16:07:29,786 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1853865330, stopped=false
2024-11-13T16:07:29,786 INFO  [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c4c650e80f58,38989,1731514048892
2024-11-13T16:07:29,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:29,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running
2024-11-13T16:07:29,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:29,788 INFO  [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:07:29,788 INFO  [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test.
2024-11-13T16:07:29,789 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277)
    at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265)
    at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036)
    at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020)
    at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
    at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.base/java.lang.reflect.Method.invoke(Method.java:568)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
    at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33)
    at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61)
    at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306)
    at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63)
    at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329)
    at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
    at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:29,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:29,789 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:29,789 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running
2024-11-13T16:07:29,789 INFO  [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c4c650e80f58,46683,1731514048941' *****
2024-11-13T16:07:29,789 INFO  [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested
2024-11-13T16:07:29,789 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HeapMemoryManager(220): Stopping
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully.
2024-11-13T16:07:29,790 INFO  [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully.
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(959): stopping server c4c650e80f58,46683,1731514048941
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c4c650e80f58:46683.
2024-11-13T16:07:29,790 DEBUG [RS:0;c4c650e80f58:46683 {}] client.AsyncConnectionImpl(264): Call stack:
    at java.base/java.lang.Thread.getStackTrace(Thread.java:1619)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187)
    at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177)
    at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229)
    at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457)
    at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155)
    at java.base/java.security.AccessController.doPrivileged(AccessController.java:399)
    at java.base/javax.security.auth.Subject.doAs(Subject.java:376)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930)
    at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322)
    at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152)
    at java.base/java.lang.Thread.run(Thread.java:840)

2024-11-13T16:07:29,790 DEBUG [RS:0;c4c650e80f58:46683 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish...
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish...
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish...
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740
2024-11-13T16:07:29,790 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close
2024-11-13T16:07:29,791 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740}
2024-11-13T16:07:29,791 DEBUG [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1351): Waiting on 1588230740
2024-11-13T16:07:29,791 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes
2024-11-13T16:07:29,791 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740
2024-11-13T16:07:29,791 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740
2024-11-13T16:07:29,791 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms
2024-11-13T16:07:29,791 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740
2024-11-13T16:07:29,791 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB
2024-11-13T16:07:29,807 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/.tmp/ns/704c3d9044a7452c9373768a680eb71d is 43, key is default/ns:d/1731514049720/Put/seqid=0
2024-11-13T16:07:29,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741837_1013 (size=5153)
2024-11-13T16:07:29,811 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741837_1013 (size=5153)
2024-11-13T16:07:29,812 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/.tmp/ns/704c3d9044a7452c9373768a680eb71d
2024-11-13T16:07:29,817 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/.tmp/ns/704c3d9044a7452c9373768a680eb71d as hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/ns/704c3d9044a7452c9373768a680eb71d
2024-11-13T16:07:29,820 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/ns/704c3d9044a7452c9373768a680eb71d, entries=2, sequenceid=6, filesize=5.0 K
2024-11-13T16:07:29,821 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false
2024-11-13T16:07:29,825 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1
2024-11-13T16:07:29,825 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint
2024-11-13T16:07:29,825 INFO  [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740
2024-11-13T16:07:29,825 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740:
Waiting for close lock at 1731514049791Running coprocessor pre-close hooks at 1731514049791Disabling compacts and flushes for region at 1731514049791Disabling writes for close at 1731514049791Obtaining lock to block concurrent updates at 1731514049791Preparing flush snapshotting stores in 1588230740 at 1731514049791Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731514049791Flushing stores of hbase:meta,,1.1588230740 at 1731514049792 (+1 ms)Flushing 1588230740/ns: creating writer at 1731514049792Flushing 1588230740/ns: appending metadata at 1731514049806 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731514049806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ce35305: reopening flushed file at 1731514049816 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=6, compaction requested=false at 1731514049821 (+5 ms)Writing region close event to WAL at 1731514049822 (+1 ms)Running coprocessor post-close hooks at 1731514049825 (+3 ms)Closed at 1731514049825
2024-11-13T16:07:29,825 DEBUG [RS_CLOSE_META-regionserver/c4c650e80f58:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740
2024-11-13T16:07:29,991 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(976): stopping server c4c650e80f58,46683,1731514048941; all regions closed.
2024-11-13T16:07:29,991 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,991 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,991 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,992 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,992 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,993 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741834_1010 (size=1152)
2024-11-13T16:07:29,994 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741834_1010 (size=1152)
2024-11-13T16:07:29,995 DEBUG [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs
2024-11-13T16:07:29,996 INFO  [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C46683%2C1731514048941.meta:.meta(num 1731514049678)
2024-11-13T16:07:29,996 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,996 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,996 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,996 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,996 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:29,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741833_1009 (size=93)
2024-11-13T16:07:29,998 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741833_1009 (size=93)
2024-11-13T16:07:30,000 DEBUG [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/oldWALs
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c4c650e80f58%2C46683%2C1731514048941:(num 1731514049317)
2024-11-13T16:07:30,000 DEBUG [RS:0;c4c650e80f58:46683 {}] ipc.AbstractRpcClient(514): Stopping rpc client
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.LeaseManager(133): Closed leases
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.ChoreService(370): Chore service for: regionserver/c4c650e80f58:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:07:30,000 INFO  [RS:0;c4c650e80f58:46683 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46683
2024-11-13T16:07:30,000 INFO  [regionserver/c4c650e80f58:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:07:30,002 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:07:30,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c4c650e80f58,46683,1731514048941
2024-11-13T16:07:30,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs
2024-11-13T16:07:30,004 INFO  [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c4c650e80f58,46683,1731514048941]
2024-11-13T16:07:30,006 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c4c650e80f58,46683,1731514048941 already deleted, retry=false
2024-11-13T16:07:30,006 INFO  [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c4c650e80f58,46683,1731514048941 expired; onlineServers=0
2024-11-13T16:07:30,006 INFO  [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c4c650e80f58,38989,1731514048892' *****
2024-11-13T16:07:30,006 INFO  [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0
2024-11-13T16:07:30,006 INFO  [M:0;c4c650e80f58:38989 {}] hbase.HBaseServerBase(455): Close async cluster connection
2024-11-13T16:07:30,006 INFO  [M:0;c4c650e80f58:38989 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service
2024-11-13T16:07:30,006 DEBUG [M:0;c4c650e80f58:38989 {}] cleaner.LogCleaner(198): Cancelling LogCleaner
2024-11-13T16:07:30,006 WARN  [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting.
2024-11-13T16:07:30,006 DEBUG [M:0;c4c650e80f58:38989 {}] cleaner.HFileCleaner(335): Stopping file delete threads
2024-11-13T16:07:30,006 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731514049084 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.large.0-1731514049084,5,FailOnTimeoutGroup]
2024-11-13T16:07:30,006 DEBUG [master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731514049084 {}] cleaner.HFileCleaner(306): Exit Thread[master/c4c650e80f58:0:becomeActiveMaster-HFileCleaner.small.0-1731514049084,5,FailOnTimeoutGroup]
2024-11-13T16:07:30,007 INFO  [M:0;c4c650e80f58:38989 {}] hbase.ChoreService(370): Chore service for: master/c4c650e80f58:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown
2024-11-13T16:07:30,007 INFO  [M:0;c4c650e80f58:38989 {}] hbase.HBaseServerBase(448): Shutdown executor service
2024-11-13T16:07:30,007 DEBUG [M:0;c4c650e80f58:38989 {}] master.HMaster(1795): Stopping service threads
2024-11-13T16:07:30,007 INFO  [M:0;c4c650e80f58:38989 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher
2024-11-13T16:07:30,007 INFO  [M:0;c4c650e80f58:38989 {}] procedure2.ProcedureExecutor(723): Stopping
2024-11-13T16:07:30,007 INFO  [M:0;c4c650e80f58:38989 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false
2024-11-13T16:07:30,007 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating.
2024-11-13T16:07:30,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master
2024-11-13T16:07:30,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase
2024-11-13T16:07:30,008 DEBUG [M:0;c4c650e80f58:38989 {}] zookeeper.ZKUtil(347): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error)
2024-11-13T16:07:30,008 WARN  [M:0;c4c650e80f58:38989 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null
2024-11-13T16:07:30,008 INFO  [M:0;c4c650e80f58:38989 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/.lastflushedseqids
2024-11-13T16:07:30,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741838_1014 (size=99)
2024-11-13T16:07:30,014 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741838_1014 (size=99)
2024-11-13T16:07:30,014 INFO  [M:0;c4c650e80f58:38989 {}] assignment.AssignmentManager(395): Stopping assignment manager
2024-11-13T16:07:30,014 INFO  [M:0;c4c650e80f58:38989 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false
2024-11-13T16:07:30,014 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes
2024-11-13T16:07:30,014 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:30,014 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:30,015 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms
2024-11-13T16:07:30,015 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:30,015 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB
2024-11-13T16:07:30,036 DEBUG [M:0;c4c650e80f58:38989 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a35f39303fff4008b9631be9693e0983 is 82, key is hbase:meta,,1/info:regioninfo/1731514049705/Put/seqid=0
2024-11-13T16:07:30,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741839_1015 (size=5672)
2024-11-13T16:07:30,041 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741839_1015 (size=5672)
2024-11-13T16:07:30,041 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a35f39303fff4008b9631be9693e0983
2024-11-13T16:07:30,059 DEBUG [M:0;c4c650e80f58:38989 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ab945a962a94d038989a34629874b61 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731514049723/Put/seqid=0
2024-11-13T16:07:30,063 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741840_1016 (size=5275)
2024-11-13T16:07:30,064 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741840_1016 (size=5275)
2024-11-13T16:07:30,064 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ab945a962a94d038989a34629874b61
2024-11-13T16:07:30,082 DEBUG [M:0;c4c650e80f58:38989 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82c6f9c719f4c0fb988346b12e44fc0 is 69, key is c4c650e80f58,46683,1731514048941/rs:state/1731514049175/Put/seqid=0
2024-11-13T16:07:30,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741841_1017 (size=5156)
2024-11-13T16:07:30,086 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741841_1017 (size=5156)
2024-11-13T16:07:30,087 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82c6f9c719f4c0fb988346b12e44fc0
2024-11-13T16:07:30,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:30,104 INFO  [RS:0;c4c650e80f58:46683 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:07:30,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46683-0x100a60b318e0001, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:30,104 INFO  [RS:0;c4c650e80f58:46683 {}] regionserver.HRegionServer(1031): Exiting; stopping=c4c650e80f58,46683,1731514048941; zookeeper connection closed.
2024-11-13T16:07:30,104 INFO  [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36153b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36153b
2024-11-13T16:07:30,105 INFO  [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete
2024-11-13T16:07:30,105 DEBUG [M:0;c4c650e80f58:38989 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18a99f5fd7ed4db7b6fabe1b236273b4 is 52, key is load_balancer_on/state:d/1731514049763/Put/seqid=0
2024-11-13T16:07:30,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741842_1018 (size=5056)
2024-11-13T16:07:30,110 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741842_1018 (size=5056)
2024-11-13T16:07:30,110 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18a99f5fd7ed4db7b6fabe1b236273b4
2024-11-13T16:07:30,115 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a35f39303fff4008b9631be9693e0983 as hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a35f39303fff4008b9631be9693e0983
2024-11-13T16:07:30,119 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a35f39303fff4008b9631be9693e0983, entries=8, sequenceid=29, filesize=5.5 K
2024-11-13T16:07:30,120 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4ab945a962a94d038989a34629874b61 as hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ab945a962a94d038989a34629874b61
2024-11-13T16:07:30,123 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4ab945a962a94d038989a34629874b61, entries=3, sequenceid=29, filesize=5.2 K
2024-11-13T16:07:30,124 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d82c6f9c719f4c0fb988346b12e44fc0 as hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d82c6f9c719f4c0fb988346b12e44fc0
2024-11-13T16:07:30,127 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d82c6f9c719f4c0fb988346b12e44fc0, entries=1, sequenceid=29, filesize=5.0 K
2024-11-13T16:07:30,128 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18a99f5fd7ed4db7b6fabe1b236273b4 as hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/18a99f5fd7ed4db7b6fabe1b236273b4
2024-11-13T16:07:30,132 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36573/user/jenkins/test-data/376f13d2-7ff6-620c-9294-14fa117c0430/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/18a99f5fd7ed4db7b6fabe1b236273b4, entries=1, sequenceid=29, filesize=4.9 K
2024-11-13T16:07:30,133 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=29, compaction requested=false
2024-11-13T16:07:30,135 INFO  [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682.
2024-11-13T16:07:30,135 DEBUG [M:0;c4c650e80f58:38989 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682:
Waiting for close lock at 1731514050014Disabling compacts and flushes for region at 1731514050014Disabling writes for close at 1731514050015 (+1 ms)Obtaining lock to block concurrent updates at 1731514050015Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731514050015Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731514050015Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731514050016 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731514050016Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731514050036 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731514050036Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731514050045 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731514050059 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731514050059Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731514050068 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731514050081 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731514050081Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731514050090 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731514050104 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731514050104Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78a66890: reopening flushed file at 1731514050115 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ded4951: reopening flushed file at 1731514050119 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8aa3af8: reopening flushed file at 1731514050123 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@446298f7: reopening flushed file at 1731514050127 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 118ms, sequenceid=29, compaction requested=false at 1731514050133 (+6 ms)Writing region close event to WAL at 1731514050134 (+1 ms)Closed at 1731514050134
2024-11-13T16:07:30,135 INFO  [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:30,135 INFO  [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:30,135 INFO  [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:30,135 INFO  [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:30,135 INFO  [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted
2024-11-13T16:07:30,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36799 is added to blk_1073741830_1006 (size=10311)
2024-11-13T16:07:30,137 INFO  [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38925 is added to blk_1073741830_1006 (size=10311)
2024-11-13T16:07:30,138 INFO  [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting.
2024-11-13T16:07:30,138 INFO  [M:0;c4c650e80f58:38989 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down.
2024-11-13T16:07:30,138 INFO  [M:0;c4c650e80f58:38989 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38989
2024-11-13T16:07:30,139 INFO  [M:0;c4c650e80f58:38989 {}] hbase.HBaseServerBase(479): Close zookeeper
2024-11-13T16:07:30,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:30,241 INFO  [M:0;c4c650e80f58:38989 {}] hbase.HBaseServerBase(486): Close table descriptors
2024-11-13T16:07:30,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38989-0x100a60b318e0000, quorum=127.0.0.1:57468, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null
2024-11-13T16:07:30,243 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5982a43f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:30,244 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@734134af{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:30,244 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:30,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3829c39e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:30,244 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@698d258d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:30,245 WARN  [BP-523714403-172.17.0.3-1731514048197 heartbeating to localhost/127.0.0.1:36573 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:07:30,245 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:07:30,245 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:07:30,245 WARN  [BP-523714403-172.17.0.3-1731514048197 heartbeating to localhost/127.0.0.1:36573 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-523714403-172.17.0.3-1731514048197 (Datanode Uuid 3994a46a-b80b-4d42-8cdd-c2a209448a38) service to localhost/127.0.0.1:36573
2024-11-13T16:07:30,246 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data3/current/BP-523714403-172.17.0.3-1731514048197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:30,246 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data4/current/BP-523714403-172.17.0.3-1731514048197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:30,246 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:07:30,248 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a35d82b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode}
2024-11-13T16:07:30,248 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@591676f4{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:30,248 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:30,248 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acdff1a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:30,248 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e4376a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:30,250 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit.
2024-11-13T16:07:30,250 WARN  [BP-523714403-172.17.0.3-1731514048197 heartbeating to localhost/127.0.0.1:36573 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted
2024-11-13T16:07:30,250 WARN  [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup]
2024-11-13T16:07:30,250 WARN  [BP-523714403-172.17.0.3-1731514048197 heartbeating to localhost/127.0.0.1:36573 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-523714403-172.17.0.3-1731514048197 (Datanode Uuid 87dc5a08-9c15-4ff6-ba75-3ece44e674e3) service to localhost/127.0.0.1:36573
2024-11-13T16:07:30,250 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data1/current/BP-523714403-172.17.0.3-1731514048197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:30,251 WARN  [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/cluster_de6cebae-4a5f-5528-a5aa-b8e0c5b3b4de/data/data2/current/BP-523714403-172.17.0.3-1731514048197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted
2024-11-13T16:07:30,251 WARN  [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func
2024-11-13T16:07:30,256 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@377e4a58{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs}
2024-11-13T16:07:30,257 INFO  [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8035060{HTTP/1.1, (http/1.1)}{localhost:0}
2024-11-13T16:07:30,257 INFO  [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging
2024-11-13T16:07:30,257 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@49428dc3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED}
2024-11-13T16:07:30,257 INFO  [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56836339{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0ab2f7ea-260f-e09e-4615-c08eecf64957/hadoop.log.dir/,STOPPED}
2024-11-13T16:07:30,263 INFO  [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers
2024-11-13T16:07:30,275 INFO  [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down
2024-11-13T16:07:30,285 INFO  [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=267 (was 229)
Potentially hanging thread: HMaster-EventLoopGroup-16-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36573 from jenkins.hfs.7
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:36573
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-44-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-42-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: HMaster-EventLoopGroup-16-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36573 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-2
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-45-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36573
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704)
	java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
	app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308)
	app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
	app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: nioEventLoopGroup-43-1
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: LeaseRenewer:jenkins@localhost:36573
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77)
	app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: globalEventExecutor-1-19
	java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method)
	java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
	java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674)
	java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460)
	app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113)
	app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)

Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36573 from jenkins
	java.base@17.0.11/java.lang.Object.wait(Native Method)
	app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042)
	app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093)

Potentially hanging thread: nioEventLoopGroup-45-3
	java.base@17.0.11/java.lang.Thread.sleep(Native Method)
	app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787)
	app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596)
	app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
	app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	java.base@17.0.11/java.lang.Thread.run(Thread.java:840)
 - Thread LEAK? -, OpenFileDescriptor=541 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=33 (was 36), ProcessCount=11 (was 11), AvailableMemoryMB=11252 (was 11262)