2024-11-24 01:46:08,920 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 01:46:08,931 main DEBUG Took 0.009571 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 01:46:08,932 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 01:46:08,932 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 01:46:08,933 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 01:46:08,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,941 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 01:46:08,953 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,955 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,955 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,955 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,957 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,958 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,960 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,960 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,961 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,962 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 01:46:08,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,964 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 01:46:08,965 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 01:46:08,966 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 01:46:08,968 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 01:46:08,968 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 01:46:08,969 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 01:46:08,970 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 01:46:08,977 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 01:46:08,980 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 01:46:08,982 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 01:46:08,982 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 01:46:08,983 main DEBUG createAppenders(={Console}) 2024-11-24 01:46:08,983 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 01:46:08,984 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 01:46:08,984 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 01:46:08,984 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 01:46:08,985 main DEBUG OutputStream closed 2024-11-24 01:46:08,985 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 01:46:08,985 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 01:46:08,985 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 01:46:09,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 01:46:09,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 01:46:09,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 01:46:09,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 01:46:09,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 01:46:09,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 01:46:09,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 01:46:09,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 01:46:09,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 01:46:09,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 01:46:09,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 01:46:09,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 01:46:09,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 01:46:09,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 01:46:09,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 01:46:09,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 01:46:09,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 01:46:09,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 01:46:09,068 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 01:46:09,068 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 01:46:09,068 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 01:46:09,069 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T01:46:09,302 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d 2024-11-24 01:46:09,305 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 01:46:09,305 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T01:46:09,313 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T01:46:09,319 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-24T01:46:09,342 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T01:46:09,386 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T01:46:09,386 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T01:46:09,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T01:46:09,411 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37, deleteOnExit=true 2024-11-24T01:46:09,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T01:46:09,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/test.cache.data in system properties and HBase conf 2024-11-24T01:46:09,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T01:46:09,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir in system properties and HBase conf 2024-11-24T01:46:09,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T01:46:09,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T01:46:09,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T01:46:09,489 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T01:46:09,588 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T01:46:09,593 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T01:46:09,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T01:46:09,594 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T01:46:09,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T01:46:09,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T01:46:09,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T01:46:09,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T01:46:09,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T01:46:09,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T01:46:09,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/nfs.dump.dir in system properties and HBase conf 2024-11-24T01:46:09,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/java.io.tmpdir in system properties and HBase conf 2024-11-24T01:46:09,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T01:46:09,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T01:46:09,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T01:46:10,614 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T01:46:10,684 INFO [Time-limited test {}] log.Log(170): Logging initialized @2438ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T01:46:10,754 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T01:46:10,812 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T01:46:10,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T01:46:10,829 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T01:46:10,830 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T01:46:10,852 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T01:46:10,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,AVAILABLE} 2024-11-24T01:46:10,858 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T01:46:11,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/java.io.tmpdir/jetty-localhost-39353-hadoop-hdfs-3_4_1-tests_jar-_-any-2308192987908944603/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T01:46:11,036 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:39353} 2024-11-24T01:46:11,037 INFO [Time-limited test {}] server.Server(415): Started @2790ms 2024-11-24T01:46:11,583 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T01:46:11,591 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T01:46:11,592 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T01:46:11,593 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T01:46:11,593 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T01:46:11,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,AVAILABLE} 2024-11-24T01:46:11,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T01:46:11,694 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/java.io.tmpdir/jetty-localhost-33671-hadoop-hdfs-3_4_1-tests_jar-_-any-12159915983811117908/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:11,695 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:33671} 2024-11-24T01:46:11,695 INFO [Time-limited test {}] server.Server(415): Started @3448ms 2024-11-24T01:46:11,740 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T01:46:11,849 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T01:46:11,855 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T01:46:11,857 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T01:46:11,857 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T01:46:11,857 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T01:46:11,859 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,AVAILABLE} 2024-11-24T01:46:11,859 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T01:46:11,960 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/java.io.tmpdir/jetty-localhost-44115-hadoop-hdfs-3_4_1-tests_jar-_-any-11753996487358312307/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:11,961 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:44115} 2024-11-24T01:46:11,962 INFO [Time-limited test {}] server.Server(415): Started @3715ms 2024-11-24T01:46:11,964 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T01:46:11,998 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T01:46:12,003 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T01:46:12,004 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T01:46:12,005 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T01:46:12,005 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T01:46:12,006 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,AVAILABLE} 2024-11-24T01:46:12,006 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T01:46:12,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/java.io.tmpdir/jetty-localhost-37311-hadoop-hdfs-3_4_1-tests_jar-_-any-11908028399726171612/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:12,112 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:37311} 2024-11-24T01:46:12,112 INFO [Time-limited test {}] server.Server(415): Started @3865ms 2024-11-24T01:46:12,114 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T01:46:12,944 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data1/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,944 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data3/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,944 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data4/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,944 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data2/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,977 WARN [Thread-136 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data5/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,980 WARN [Thread-137 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data6/current/BP-2032936213-172.17.0.2-1732412770132/current, will proceed with Du for space computation calculation, 2024-11-24T01:46:12,989 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T01:46:12,990 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T01:46:13,005 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T01:46:13,039 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a8f632b9bb805fa with lease ID 0x3c18fa5b17b50fb8: Processing first storage report for DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1 from datanode DatanodeRegistration(127.0.0.1:33251, datanodeUuid=ca64ab46-0690-4a7a-86ce-ac2bffc292b2, infoPort=38645, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a8f632b9bb805fa with lease ID 0x3c18fa5b17b50fb8: from storage DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1 node DatanodeRegistration(127.0.0.1:33251, datanodeUuid=ca64ab46-0690-4a7a-86ce-ac2bffc292b2, infoPort=38645, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7eb39911764de0f0 with lease ID 0x3c18fa5b17b50fb9: Processing first storage report for DS-d62473c1-5a99-42df-bfc9-84a75470c6fd from datanode DatanodeRegistration(127.0.0.1:39381, datanodeUuid=fc7c409e-68ec-4a95-97d6-5e186e903867, infoPort=38873, infoSecurePort=0, ipcPort=42853, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7eb39911764de0f0 with lease ID 0x3c18fa5b17b50fb9: from storage DS-d62473c1-5a99-42df-bfc9-84a75470c6fd node DatanodeRegistration(127.0.0.1:39381, datanodeUuid=fc7c409e-68ec-4a95-97d6-5e186e903867, infoPort=38873, infoSecurePort=0, ipcPort=42853, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,040 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe745c32e7a0aca8 with lease ID 0x3c18fa5b17b50fb7: Processing first storage report for DS-086d5bc5-16c3-476a-8d39-9b25c17950c7 from datanode DatanodeRegistration(127.0.0.1:39823, datanodeUuid=ed5d09ff-5535-42c9-a78d-b432c1bb6737, infoPort=33015, infoSecurePort=0, ipcPort=32809, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe745c32e7a0aca8 with lease ID 0x3c18fa5b17b50fb7: from storage DS-086d5bc5-16c3-476a-8d39-9b25c17950c7 node DatanodeRegistration(127.0.0.1:39823, datanodeUuid=ed5d09ff-5535-42c9-a78d-b432c1bb6737, infoPort=33015, infoSecurePort=0, ipcPort=32809, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3a8f632b9bb805fa with lease ID 0x3c18fa5b17b50fb8: Processing first storage report for DS-55dcbdb2-d04a-46bf-b765-4acfd06bdf12 from datanode DatanodeRegistration(127.0.0.1:33251, datanodeUuid=ca64ab46-0690-4a7a-86ce-ac2bffc292b2, infoPort=38645, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3a8f632b9bb805fa with lease ID 0x3c18fa5b17b50fb8: from storage DS-55dcbdb2-d04a-46bf-b765-4acfd06bdf12 node DatanodeRegistration(127.0.0.1:33251, datanodeUuid=ca64ab46-0690-4a7a-86ce-ac2bffc292b2, infoPort=38645, infoSecurePort=0, ipcPort=35539, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7eb39911764de0f0 with lease ID 0x3c18fa5b17b50fb9: Processing first storage report for DS-badca1a7-3d0c-4963-a387-6c916566addc from datanode DatanodeRegistration(127.0.0.1:39381, datanodeUuid=fc7c409e-68ec-4a95-97d6-5e186e903867, infoPort=38873, infoSecurePort=0, ipcPort=42853, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7eb39911764de0f0 with lease ID 0x3c18fa5b17b50fb9: from storage DS-badca1a7-3d0c-4963-a387-6c916566addc node DatanodeRegistration(127.0.0.1:39381, datanodeUuid=fc7c409e-68ec-4a95-97d6-5e186e903867, infoPort=38873, infoSecurePort=0, ipcPort=42853, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,042 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbe745c32e7a0aca8 with lease ID 0x3c18fa5b17b50fb7: Processing first storage report for DS-48a0178f-ac30-4d75-a495-fc6755e841b8 from datanode DatanodeRegistration(127.0.0.1:39823, datanodeUuid=ed5d09ff-5535-42c9-a78d-b432c1bb6737, infoPort=33015, infoSecurePort=0, ipcPort=32809, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132) 2024-11-24T01:46:13,042 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbe745c32e7a0aca8 with lease ID 0x3c18fa5b17b50fb7: from storage DS-48a0178f-ac30-4d75-a495-fc6755e841b8 node DatanodeRegistration(127.0.0.1:39823, datanodeUuid=ed5d09ff-5535-42c9-a78d-b432c1bb6737, infoPort=33015, infoSecurePort=0, ipcPort=32809, storageInfo=lv=-57;cid=testClusterID;nsid=223200408;c=1732412770132), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T01:46:13,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d 2024-11-24T01:46:13,202 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/zookeeper_0, clientPort=60515, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T01:46:13,211 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60515 2024-11-24T01:46:13,226 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:13,230 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:13,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741825_1001 (size=7) 2024-11-24T01:46:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741825_1001 (size=7) 2024-11-24T01:46:13,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741825_1001 (size=7) 2024-11-24T01:46:13,813 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 with version=8 2024-11-24T01:46:13,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/hbase-staging 2024-11-24T01:46:14,091 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T01:46:14,099 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,099 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,103 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T01:46:14,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,104 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T01:46:14,244 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T01:46:14,294 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T01:46:14,302 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T01:46:14,305 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T01:46:14,327 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 24905 (auto-detected) 2024-11-24T01:46:14,328 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T01:46:14,344 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37581 2024-11-24T01:46:14,362 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37581 connecting to ZooKeeper ensemble=127.0.0.1:60515 2024-11-24T01:46:14,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:375810x0, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T01:46:14,495 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37581-0x1016bcea5fe0000 connected 2024-11-24T01:46:14,606 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,610 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:14,623 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352, hbase.cluster.distributed=false 2024-11-24T01:46:14,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T01:46:14,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-24T01:46:14,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37581 2024-11-24T01:46:14,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37581 2024-11-24T01:46:14,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-24T01:46:14,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37581 2024-11-24T01:46:14,749 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T01:46:14,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,750 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T01:46:14,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,751 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T01:46:14,753 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T01:46:14,755 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T01:46:14,756 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40811 2024-11-24T01:46:14,758 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40811 connecting to ZooKeeper ensemble=127.0.0.1:60515 2024-11-24T01:46:14,759 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,763 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:408110x0, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T01:46:14,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:408110x0, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:14,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40811-0x1016bcea5fe0001 connected 2024-11-24T01:46:14,799 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T01:46:14,807 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T01:46:14,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T01:46:14,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T01:46:14,814 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40811 2024-11-24T01:46:14,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40811 2024-11-24T01:46:14,815 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40811 2024-11-24T01:46:14,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40811 2024-11-24T01:46:14,816 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40811 2024-11-24T01:46:14,831 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T01:46:14,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,832 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T01:46:14,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,832 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T01:46:14,833 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T01:46:14,833 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T01:46:14,834 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33283 2024-11-24T01:46:14,836 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33283 connecting to ZooKeeper ensemble=127.0.0.1:60515 2024-11-24T01:46:14,837 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,838 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332830x0, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T01:46:14,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332830x0, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:14,868 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33283-0x1016bcea5fe0002 connected 2024-11-24T01:46:14,868 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T01:46:14,869 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T01:46:14,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T01:46:14,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T01:46:14,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33283 2024-11-24T01:46:14,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33283 2024-11-24T01:46:14,875 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33283 2024-11-24T01:46:14,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33283 2024-11-24T01:46:14,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33283 2024-11-24T01:46:14,892 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a67e7d17e593:0 server-side Connection retries=45 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T01:46:14,892 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T01:46:14,893 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T01:46:14,893 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43635 2024-11-24T01:46:14,895 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43635 connecting to ZooKeeper ensemble=127.0.0.1:60515 2024-11-24T01:46:14,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,899 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:14,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436350x0, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T01:46:14,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436350x0, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:14,912 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43635-0x1016bcea5fe0003 connected 2024-11-24T01:46:14,913 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T01:46:14,913 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T01:46:14,914 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T01:46:14,916 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T01:46:14,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-24T01:46:14,917 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43635 2024-11-24T01:46:14,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43635 2024-11-24T01:46:14,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-24T01:46:14,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-24T01:46:14,934 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a67e7d17e593:37581 2024-11-24T01:46:14,935 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a67e7d17e593,37581,1732412773942 2024-11-24T01:46:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,952 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a67e7d17e593,37581,1732412773942 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:14,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:14,979 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T01:46:14,980 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a67e7d17e593,37581,1732412773942 from backup master directory 2024-11-24T01:46:14,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a67e7d17e593,37581,1732412773942 2024-11-24T01:46:14,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T01:46:14,992 WARN [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T01:46:14,993 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a67e7d17e593,37581,1732412773942 2024-11-24T01:46:14,995 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T01:46:14,996 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T01:46:15,049 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/hbase.id] with ID: 7e44e35e-5659-455a-a56b-5baa5947e9b1 2024-11-24T01:46:15,049 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/.tmp/hbase.id 2024-11-24T01:46:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741826_1002 (size=42) 2024-11-24T01:46:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741826_1002 (size=42) 2024-11-24T01:46:15,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741826_1002 (size=42) 2024-11-24T01:46:15,064 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/.tmp/hbase.id]:[hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/hbase.id] 2024-11-24T01:46:15,110 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T01:46:15,114 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T01:46:15,132 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-24T01:46:15,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741827_1003 (size=196) 2024-11-24T01:46:15,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741827_1003 (size=196) 2024-11-24T01:46:15,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741827_1003 (size=196) 2024-11-24T01:46:15,186 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T01:46:15,188 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T01:46:15,193 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:15,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741828_1004 (size=1189) 2024-11-24T01:46:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741828_1004 (size=1189) 2024-11-24T01:46:15,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741828_1004 (size=1189) 2024-11-24T01:46:15,241 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store 2024-11-24T01:46:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741829_1005 (size=34) 2024-11-24T01:46:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741829_1005 (size=34) 2024-11-24T01:46:15,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741829_1005 (size=34) 2024-11-24T01:46:15,266 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T01:46:15,270 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:15,272 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T01:46:15,272 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:15,272 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:15,274 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T01:46:15,275 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:15,275 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:15,276 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732412775272Disabling compacts and flushes for region at 1732412775272Disabling writes for close at 1732412775275 (+3 ms)Writing region close event to WAL at 1732412775275Closed at 1732412775275 2024-11-24T01:46:15,278 WARN [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/.initializing 2024-11-24T01:46:15,279 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/WALs/a67e7d17e593,37581,1732412773942 2024-11-24T01:46:15,286 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:15,302 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C37581%2C1732412773942, suffix=, logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/WALs/a67e7d17e593,37581,1732412773942, archiveDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/oldWALs, maxLogs=10 2024-11-24T01:46:15,328 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/WALs/a67e7d17e593,37581,1732412773942/a67e7d17e593%2C37581%2C1732412773942.1732412775306, exclude list is [], retry=0 2024-11-24T01:46:15,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:15,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:15,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:15,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:15,353 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T01:46:15,385 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-24T01:46:15,445 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/WALs/a67e7d17e593,37581,1732412773942/a67e7d17e593%2C37581%2C1732412773942.1732412775306 2024-11-24T01:46:15,445 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:15,446 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:15,446 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:15,449 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,449 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,481 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T01:46:15,505 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:15,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:15,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,512 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T01:46:15,513 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:15,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:15,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T01:46:15,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:15,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:15,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T01:46:15,520 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:15,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:15,522 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,525 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,526 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,531 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,532 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,536 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T01:46:15,539 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T01:46:15,543 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:15,544 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62756604, jitterRate=-0.0648537278175354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T01:46:15,551 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732412775460Initializing all the Stores at 1732412775462 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412775463 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412775464 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412775464Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412775464Cleaning up temporary data from old regions at 1732412775532 (+68 ms)Region opened successfully at 1732412775551 (+19 ms) 2024-11-24T01:46:15,552 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T01:46:15,586 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285cd7b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T01:46:15,616 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T01:46:15,625 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T01:46:15,626 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T01:46:15,628 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T01:46:15,629 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T01:46:15,633 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T01:46:15,634 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T01:46:15,661 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T01:46:15,670 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T01:46:15,716 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T01:46:15,719 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T01:46:15,721 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T01:46:15,733 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T01:46:15,735 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T01:46:15,738 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T01:46:15,749 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T01:46:15,751 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T01:46:15,761 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T01:46:15,779 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T01:46:15,785 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T01:46:15,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:15,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:15,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:15,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:15,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,798 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a67e7d17e593,37581,1732412773942, sessionid=0x1016bcea5fe0000, setting cluster-up flag (Was=false) 2024-11-24T01:46:15,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,853 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T01:46:15,856 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67e7d17e593,37581,1732412773942 2024-11-24T01:46:15,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:15,902 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T01:46:15,905 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a67e7d17e593,37581,1732412773942 2024-11-24T01:46:15,912 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T01:46:15,923 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(746): ClusterId : 7e44e35e-5659-455a-a56b-5baa5947e9b1 2024-11-24T01:46:15,923 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(746): ClusterId : 7e44e35e-5659-455a-a56b-5baa5947e9b1 2024-11-24T01:46:15,923 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(746): ClusterId : 7e44e35e-5659-455a-a56b-5baa5947e9b1 2024-11-24T01:46:15,925 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T01:46:15,925 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T01:46:15,925 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T01:46:15,946 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T01:46:15,946 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T01:46:15,946 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T01:46:15,946 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T01:46:15,946 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T01:46:15,946 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T01:46:15,962 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T01:46:15,962 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T01:46:15,962 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T01:46:15,962 DEBUG [RS:1;a67e7d17e593:33283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c705182, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T01:46:15,962 DEBUG [RS:0;a67e7d17e593:40811 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650d896f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T01:46:15,962 DEBUG [RS:2;a67e7d17e593:43635 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe6ed2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a67e7d17e593/172.17.0.2:0 2024-11-24T01:46:15,977 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a67e7d17e593:40811 2024-11-24T01:46:15,978 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a67e7d17e593:43635 2024-11-24T01:46:15,980 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a67e7d17e593:33283 2024-11-24T01:46:15,980 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T01:46:15,980 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T01:46:15,980 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T01:46:15,980 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T01:46:15,980 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T01:46:15,980 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T01:46:15,980 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T01:46:15,981 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T01:46:15,981 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T01:46:15,984 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=43635, startcode=1732412774891 2024-11-24T01:46:15,984 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=40811, startcode=1732412774716 2024-11-24T01:46:15,984 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=33283, startcode=1732412774831 2024-11-24T01:46:15,989 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T01:46:15,996 DEBUG [RS:2;a67e7d17e593:43635 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T01:46:15,996 DEBUG [RS:1;a67e7d17e593:33283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T01:46:15,996 DEBUG [RS:0;a67e7d17e593:40811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T01:46:15,998 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T01:46:16,004 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T01:46:16,010 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a67e7d17e593,37581,1732412773942 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T01:46:16,018 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T01:46:16,019 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T01:46:16,019 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T01:46:16,019 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a67e7d17e593:0, corePoolSize=5, maxPoolSize=5 2024-11-24T01:46:16,019 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a67e7d17e593:0, corePoolSize=10, maxPoolSize=10 2024-11-24T01:46:16,020 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,020 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T01:46:16,020 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,026 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T01:46:16,026 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T01:46:16,029 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732412806029 2024-11-24T01:46:16,030 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T01:46:16,030 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T01:46:16,031 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T01:46:16,031 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35337, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T01:46:16,032 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T01:46:16,034 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:16,034 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T01:46:16,035 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T01:46:16,036 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T01:46:16,036 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T01:46:16,036 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T01:46:16,036 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T01:46:16,038 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,043 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T01:46:16,043 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T01:46:16,044 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T01:46:16,044 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-24T01:46:16,044 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T01:46:16,047 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T01:46:16,047 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T01:46:16,068 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732412776067,5,FailOnTimeoutGroup] 2024-11-24T01:46:16,068 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732412776068,5,FailOnTimeoutGroup] 2024-11-24T01:46:16,069 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,069 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T01:46:16,070 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,070 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741831_1007 (size=1321) 2024-11-24T01:46:16,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741831_1007 (size=1321) 2024-11-24T01:46:16,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741831_1007 (size=1321) 2024-11-24T01:46:16,075 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T01:46:16,075 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 2024-11-24T01:46:16,080 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T01:46:16,080 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T01:46:16,080 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-24T01:46:16,080 WARN [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T01:46:16,080 WARN [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T01:46:16,080 WARN [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-24T01:46:16,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741832_1008 (size=32) 2024-11-24T01:46:16,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741832_1008 (size=32) 2024-11-24T01:46:16,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741832_1008 (size=32) 2024-11-24T01:46:16,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:16,094 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T01:46:16,097 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T01:46:16,097 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:16,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:16,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T01:46:16,101 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T01:46:16,101 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:16,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:16,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T01:46:16,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T01:46:16,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:16,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:16,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T01:46:16,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T01:46:16,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:16,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:16,111 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T01:46:16,113 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740 2024-11-24T01:46:16,114 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740 2024-11-24T01:46:16,117 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T01:46:16,117 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T01:46:16,118 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T01:46:16,121 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T01:46:16,125 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:16,127 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58881427, jitterRate=-0.12259836494922638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T01:46:16,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732412776091Initializing all the Stores at 1732412776093 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412776093Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412776093Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412776093Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412776093Cleaning up temporary data from old regions at 1732412776117 (+24 ms)Region opened successfully at 1732412776132 (+15 ms) 2024-11-24T01:46:16,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T01:46:16,132 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T01:46:16,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T01:46:16,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T01:46:16,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T01:46:16,134 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T01:46:16,134 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732412776132Disabling compacts and flushes for region at 1732412776132Disabling writes for close at 1732412776133 (+1 ms)Writing region close event to WAL at 1732412776134 (+1 ms)Closed at 1732412776134 2024-11-24T01:46:16,137 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T01:46:16,138 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T01:46:16,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T01:46:16,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T01:46:16,154 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T01:46:16,182 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=43635, startcode=1732412774891 2024-11-24T01:46:16,182 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=40811, startcode=1732412774716 2024-11-24T01:46:16,182 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(2659): reportForDuty to master=a67e7d17e593,37581,1732412773942 with port=33283, startcode=1732412774831 2024-11-24T01:46:16,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,188 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(517): Registering regionserver=a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,197 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,197 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 2024-11-24T01:46:16,197 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(517): Registering regionserver=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,197 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34675 2024-11-24T01:46:16,197 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T01:46:16,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,200 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 2024-11-24T01:46:16,200 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37581 {}] master.ServerManager(517): Registering regionserver=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,200 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34675 2024-11-24T01:46:16,200 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T01:46:16,203 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 2024-11-24T01:46:16,203 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34675 2024-11-24T01:46:16,203 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T01:46:16,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T01:46:16,254 DEBUG [RS:2;a67e7d17e593:43635 {}] zookeeper.ZKUtil(111): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,255 WARN [RS:2;a67e7d17e593:43635 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T01:46:16,255 INFO [RS:2;a67e7d17e593:43635 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:16,255 DEBUG [RS:0;a67e7d17e593:40811 {}] zookeeper.ZKUtil(111): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,256 WARN [RS:0;a67e7d17e593:40811 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T01:46:16,256 DEBUG [RS:1;a67e7d17e593:33283 {}] zookeeper.ZKUtil(111): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,256 DEBUG [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,256 INFO [RS:0;a67e7d17e593:40811 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:16,256 WARN [RS:1;a67e7d17e593:33283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T01:46:16,256 INFO [RS:1;a67e7d17e593:33283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:16,257 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,257 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a67e7d17e593,40811,1732412774716] 2024-11-24T01:46:16,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a67e7d17e593,33283,1732412774831] 2024-11-24T01:46:16,259 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a67e7d17e593,43635,1732412774891] 2024-11-24T01:46:16,283 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T01:46:16,283 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T01:46:16,283 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T01:46:16,300 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T01:46:16,300 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T01:46:16,301 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T01:46:16,304 WARN [a67e7d17e593:37581 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T01:46:16,305 INFO [RS:2;a67e7d17e593:43635 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T01:46:16,306 INFO [RS:0;a67e7d17e593:40811 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T01:46:16,306 INFO [RS:1;a67e7d17e593:33283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T01:46:16,306 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,306 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,306 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,307 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T01:46:16,307 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T01:46:16,307 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T01:46:16,313 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T01:46:16,313 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T01:46:16,313 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T01:46:16,315 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,315 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,315 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,315 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,315 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0, corePoolSize=2, maxPoolSize=2 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,316 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,317 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,317 DEBUG [RS:2;a67e7d17e593:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,317 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a67e7d17e593:0, corePoolSize=1, maxPoolSize=1 2024-11-24T01:46:16,317 DEBUG [RS:1;a67e7d17e593:33283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,317 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,317 DEBUG [RS:0;a67e7d17e593:40811 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a67e7d17e593:0, corePoolSize=3, maxPoolSize=3 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,337 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,43635,1732412774891-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,33283,1732412774831-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,338 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,40811,1732412774716-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T01:46:16,363 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T01:46:16,363 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T01:46:16,363 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T01:46:16,365 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,40811,1732412774716-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,365 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,33283,1732412774831-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,365 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,43635,1732412774891-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,365 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,365 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,365 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,366 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.Replication(171): a67e7d17e593,43635,1732412774891 started 2024-11-24T01:46:16,366 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.Replication(171): a67e7d17e593,33283,1732412774831 started 2024-11-24T01:46:16,366 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.Replication(171): a67e7d17e593,40811,1732412774716 started 2024-11-24T01:46:16,390 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,390 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,390 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:16,391 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1482): Serving as a67e7d17e593,33283,1732412774831, RpcServer on a67e7d17e593/172.17.0.2:33283, sessionid=0x1016bcea5fe0002 2024-11-24T01:46:16,391 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1482): Serving as a67e7d17e593,43635,1732412774891, RpcServer on a67e7d17e593/172.17.0.2:43635, sessionid=0x1016bcea5fe0003 2024-11-24T01:46:16,391 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1482): Serving as a67e7d17e593,40811,1732412774716, RpcServer on a67e7d17e593/172.17.0.2:40811, sessionid=0x1016bcea5fe0001 2024-11-24T01:46:16,391 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T01:46:16,391 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T01:46:16,391 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T01:46:16,391 DEBUG [RS:1;a67e7d17e593:33283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,392 DEBUG [RS:2;a67e7d17e593:43635 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,391 DEBUG [RS:0;a67e7d17e593:40811 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,392 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,33283,1732412774831' 2024-11-24T01:46:16,392 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,40811,1732412774716' 2024-11-24T01:46:16,392 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,43635,1732412774891' 2024-11-24T01:46:16,392 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T01:46:16,392 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T01:46:16,392 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T01:46:16,393 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T01:46:16,393 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T01:46:16,393 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T01:46:16,393 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T01:46:16,393 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T01:46:16,393 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T01:46:16,393 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T01:46:16,393 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T01:46:16,393 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T01:46:16,393 DEBUG [RS:1;a67e7d17e593:33283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,393 DEBUG [RS:2;a67e7d17e593:43635 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a67e7d17e593,43635,1732412774891 2024-11-24T01:46:16,393 DEBUG [RS:0;a67e7d17e593:40811 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a67e7d17e593,40811,1732412774716 2024-11-24T01:46:16,393 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,33283,1732412774831' 2024-11-24T01:46:16,393 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,43635,1732412774891' 2024-11-24T01:46:16,393 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a67e7d17e593,40811,1732412774716' 2024-11-24T01:46:16,394 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T01:46:16,394 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T01:46:16,394 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T01:46:16,394 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T01:46:16,394 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T01:46:16,394 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T01:46:16,395 DEBUG [RS:1;a67e7d17e593:33283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T01:46:16,395 INFO [RS:1;a67e7d17e593:33283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T01:46:16,395 INFO [RS:1;a67e7d17e593:33283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T01:46:16,395 DEBUG [RS:0;a67e7d17e593:40811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T01:46:16,395 DEBUG [RS:2;a67e7d17e593:43635 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T01:46:16,395 INFO [RS:2;a67e7d17e593:43635 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T01:46:16,395 INFO [RS:0;a67e7d17e593:40811 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T01:46:16,395 INFO [RS:2;a67e7d17e593:43635 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T01:46:16,395 INFO [RS:0;a67e7d17e593:40811 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T01:46:16,499 INFO [RS:2;a67e7d17e593:43635 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:16,499 INFO [RS:1;a67e7d17e593:33283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:16,499 INFO [RS:0;a67e7d17e593:40811 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:16,502 INFO [RS:0;a67e7d17e593:40811 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C40811%2C1732412774716, suffix=, logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716, archiveDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs, maxLogs=32 2024-11-24T01:46:16,502 INFO [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C33283%2C1732412774831, suffix=, logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831, archiveDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs, maxLogs=32 2024-11-24T01:46:16,502 INFO [RS:2;a67e7d17e593:43635 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C43635%2C1732412774891, suffix=, logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,43635,1732412774891, archiveDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs, maxLogs=32 2024-11-24T01:46:16,519 DEBUG [RS:2;a67e7d17e593:43635 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,43635,1732412774891/a67e7d17e593%2C43635%2C1732412774891.1732412776506, exclude list is [], retry=0 2024-11-24T01:46:16,522 DEBUG [RS:0;a67e7d17e593:40811 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716/a67e7d17e593%2C40811%2C1732412774716.1732412776506, exclude list is [], retry=0 2024-11-24T01:46:16,523 DEBUG [RS:1;a67e7d17e593:33283 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831/a67e7d17e593%2C33283%2C1732412774831.1732412776506, exclude list is [], retry=0 2024-11-24T01:46:16,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:16,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:16,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:16,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:16,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:16,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:16,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:16,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:16,528 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:16,561 INFO [RS:2;a67e7d17e593:43635 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,43635,1732412774891/a67e7d17e593%2C43635%2C1732412774891.1732412776506 2024-11-24T01:46:16,562 DEBUG [RS:2;a67e7d17e593:43635 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:16,571 INFO [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831/a67e7d17e593%2C33283%2C1732412774831.1732412776506 2024-11-24T01:46:16,573 DEBUG [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:16,579 INFO [RS:0;a67e7d17e593:40811 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716/a67e7d17e593%2C40811%2C1732412774716.1732412776506 2024-11-24T01:46:16,579 DEBUG [RS:0;a67e7d17e593:40811 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:16,808 DEBUG [a67e7d17e593:37581 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T01:46:16,818 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(204): Hosts are {a67e7d17e593=0} racks are {/default-rack=0} 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T01:46:16,825 INFO [a67e7d17e593:37581 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T01:46:16,825 INFO [a67e7d17e593:37581 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T01:46:16,825 INFO [a67e7d17e593:37581 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T01:46:16,825 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T01:46:16,831 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:16,837 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67e7d17e593,33283,1732412774831, state=OPENING 2024-11-24T01:46:16,883 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T01:46:16,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:16,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:16,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:16,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:16,896 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:16,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:16,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:16,897 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:16,899 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T01:46:16,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a67e7d17e593,33283,1732412774831}] 2024-11-24T01:46:17,075 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T01:46:17,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T01:46:17,089 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T01:46:17,089 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:17,090 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T01:46:17,093 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a67e7d17e593%2C33283%2C1732412774831.meta, suffix=.meta, logDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831, archiveDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs, maxLogs=32 2024-11-24T01:46:17,107 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831/a67e7d17e593%2C33283%2C1732412774831.meta.1732412777094.meta, exclude list is [], retry=0 2024-11-24T01:46:17,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:17,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:17,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:17,139 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,33283,1732412774831/a67e7d17e593%2C33283%2C1732412774831.meta.1732412777094.meta 2024-11-24T01:46:17,140 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:17,140 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:17,142 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T01:46:17,144 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T01:46:17,150 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T01:46:17,154 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T01:46:17,155 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:17,155 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T01:46:17,155 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T01:46:17,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T01:46:17,160 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T01:46:17,160 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:17,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:17,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T01:46:17,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T01:46:17,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:17,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:17,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T01:46:17,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T01:46:17,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:17,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:17,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T01:46:17,167 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T01:46:17,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:17,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T01:46:17,168 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T01:46:17,169 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740 2024-11-24T01:46:17,172 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740 2024-11-24T01:46:17,174 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T01:46:17,174 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T01:46:17,175 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T01:46:17,178 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T01:46:17,179 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61448937, jitterRate=-0.08433948457241058}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T01:46:17,179 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T01:46:17,180 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732412777155Writing region info on filesystem at 1732412777156 (+1 ms)Initializing all the Stores at 1732412777157 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412777158 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412777158Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412777158Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732412777158Cleaning up temporary data from old regions at 1732412777174 (+16 ms)Running coprocessor post-open hooks at 1732412777180 (+6 ms)Region opened successfully at 1732412777180 2024-11-24T01:46:17,187 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732412777067 2024-11-24T01:46:17,197 DEBUG [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T01:46:17,198 INFO [RS_OPEN_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T01:46:17,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:17,203 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a67e7d17e593,33283,1732412774831, state=OPEN 2024-11-24T01:46:17,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T01:46:17,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T01:46:17,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T01:46:17,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T01:46:17,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:17,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:17,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:17,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T01:46:17,237 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:17,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T01:46:17,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a67e7d17e593,33283,1732412774831 in 336 msec 2024-11-24T01:46:17,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T01:46:17,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1030 sec 2024-11-24T01:46:17,253 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T01:46:17,253 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T01:46:17,273 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T01:46:17,274 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,33283,1732412774831, seqNum=-1] 2024-11-24T01:46:17,298 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T01:46:17,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47163, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T01:46:17,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4010 sec 2024-11-24T01:46:17,346 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732412777346, completionTime=-1 2024-11-24T01:46:17,349 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T01:46:17,350 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T01:46:17,383 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T01:46:17,383 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732412837383 2024-11-24T01:46:17,384 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732412897383 2024-11-24T01:46:17,384 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-11-24T01:46:17,385 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T01:46:17,393 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,393 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,393 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,395 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a67e7d17e593:37581, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,395 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,396 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,401 DEBUG [master/a67e7d17e593:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T01:46:17,421 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.428sec 2024-11-24T01:46:17,423 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T01:46:17,424 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T01:46:17,425 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T01:46:17,425 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T01:46:17,425 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T01:46:17,426 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T01:46:17,426 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T01:46:17,430 DEBUG [master/a67e7d17e593:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T01:46:17,431 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T01:46:17,432 INFO [master/a67e7d17e593:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a67e7d17e593,37581,1732412773942-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:17,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8a5a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T01:46:17,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a67e7d17e593,37581,-1 for getting cluster id 2024-11-24T01:46:17,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T01:46:17,459 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7e44e35e-5659-455a-a56b-5baa5947e9b1' 2024-11-24T01:46:17,461 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T01:46:17,461 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7e44e35e-5659-455a-a56b-5baa5947e9b1" 2024-11-24T01:46:17,462 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d5b2929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T01:46:17,462 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a67e7d17e593,37581,-1] 2024-11-24T01:46:17,464 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T01:46:17,466 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:17,467 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T01:46:17,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffad1df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T01:46:17,471 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T01:46:17,480 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a67e7d17e593,33283,1732412774831, seqNum=-1] 2024-11-24T01:46:17,481 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T01:46:17,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T01:46:17,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a67e7d17e593,37581,1732412773942 2024-11-24T01:46:17,509 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:34675/hbase 2024-11-24T01:46:17,525 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=368, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=7903 2024-11-24T01:46:17,552 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:17,556 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:17,557 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:17,562 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63764519, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-63764519, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:17,577 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63764519/hregion-63764519.1732412777564, exclude list is [], retry=0 2024-11-24T01:46:17,582 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:17,582 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:17,583 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:17,622 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63764519/hregion-63764519.1732412777564 2024-11-24T01:46:17,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:17,623 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8aad8033cb3b2dc456833057ce38715b, NAME => 'testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:17,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741838_1014 (size=64) 2024-11-24T01:46:17,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741838_1014 (size=64) 2024-11-24T01:46:17,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741838_1014 (size=64) 2024-11-24T01:46:17,637 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:17,639 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,641 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8aad8033cb3b2dc456833057ce38715b columnFamilyName a 2024-11-24T01:46:17,641 DEBUG [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:17,642 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] regionserver.HStore(327): Store=8aad8033cb3b2dc456833057ce38715b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:17,643 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,644 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,645 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,646 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,646 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,649 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:17,652 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:17,653 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8aad8033cb3b2dc456833057ce38715b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59553856, jitterRate=-0.1125783920288086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:17,655 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8aad8033cb3b2dc456833057ce38715b: Writing region info on filesystem at 1732412777637Initializing all the Stores at 1732412777638 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412777638Cleaning up temporary data from old regions at 1732412777646 (+8 ms)Region opened successfully at 1732412777655 (+9 ms) 2024-11-24T01:46:17,655 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8aad8033cb3b2dc456833057ce38715b, disabling compactions & flushes 2024-11-24T01:46:17,655 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. 2024-11-24T01:46:17,655 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. 2024-11-24T01:46:17,655 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. after waiting 0 ms 2024-11-24T01:46:17,655 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. 2024-11-24T01:46:17,656 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. 2024-11-24T01:46:17,656 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8aad8033cb3b2dc456833057ce38715b: Waiting for close lock at 1732412777655Disabling compacts and flushes for region at 1732412777655Disabling writes for close at 1732412777655Writing region close event to WAL at 1732412777656 (+1 ms)Closed at 1732412777656 2024-11-24T01:46:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741837_1013 (size=175) 2024-11-24T01:46:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741837_1013 (size=175) 2024-11-24T01:46:17,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741837_1013 (size=175) 2024-11-24T01:46:17,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:17,668 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63764519:(num 1732412777564) 2024-11-24T01:46:17,670 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T01:46:17,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741839_1015 (size=469) 2024-11-24T01:46:17,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741839_1015 (size=469) 2024-11-24T01:46:17,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741839_1015 (size=469) 2024-11-24T01:46:17,718 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T01:46:17,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741840_1016 (size=322) 2024-11-24T01:46:17,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741840_1016 (size=322) 2024-11-24T01:46:17,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741840_1016 (size=322) 2024-11-24T01:46:17,778 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1, size=469 (469bytes) 2024-11-24T01:46:17,779 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T01:46:17,779 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T01:46:17,779 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1 2024-11-24T01:46:17,784 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1 after 3ms 2024-11-24T01:46:17,813 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:17,814 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1 took 37ms 2024-11-24T01:46:17,818 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1 so closing down 2024-11-24T01:46:17,819 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:17,821 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T01:46:17,845 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T01:46:17,846 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:17,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741841_1017 (size=469) 2024-11-24T01:46:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741841_1017 (size=469) 2024-11-24T01:46:17,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741841_1017 (size=469) 2024-11-24T01:46:17,860 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:17,863 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002 2024-11-24T01:46:17,867 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T01:46:17,867 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1, journal: Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1, size=469 (469bytes) at 1732412777779Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1 so closing down at 1732412777819 (+40 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp at 1732412777845 (+26 ms)3 split writer threads finished at 1732412777846 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732412777860 (+14 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002 at 1732412777863 (+3 ms)Processed 2 edits across 1 Regions in 48 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732412777867 (+4 ms) 2024-11-24T01:46:17,880 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2, size=322 (322bytes) 2024-11-24T01:46:17,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2 2024-11-24T01:46:17,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2 after 1ms 2024-11-24T01:46:17,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:17,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2 took 28ms 2024-11-24T01:46:17,911 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2 so closing down 2024-11-24T01:46:17,911 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:17,914 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T01:46:17,939 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T01:46:17,939 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:17,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741842_1018 (size=322) 2024-11-24T01:46:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741842_1018 (size=322) 2024-11-24T01:46:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741842_1018 (size=322) 2024-11-24T01:46:17,949 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:17,981 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:17,984 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-24T01:46:17,986 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T01:46:17,986 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2, journal: Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2, size=322 (322bytes) at 1732412777880Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2 so closing down at 1732412777911 (+31 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002-wal-2.temp at 1732412777939 (+28 ms)3 split writer threads finished at 1732412777940 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732412777949 (+9 ms)Processed 1 edits across 1 Regions in 76 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732412777986 (+37 ms) 2024-11-24T01:46:17,986 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:17,989 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:18,002 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal.1732412777990, exclude list is [], retry=0 2024-11-24T01:46:18,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:18,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:18,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:18,034 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal.1732412777990 2024-11-24T01:46:18,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8aad8033cb3b2dc456833057ce38715b, NAME => 'testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,035 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,038 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,039 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8aad8033cb3b2dc456833057ce38715b columnFamilyName a 2024-11-24T01:46:18,039 DEBUG [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,041 INFO [StoreOpener-8aad8033cb3b2dc456833057ce38715b-1 {}] regionserver.HStore(327): Store=8aad8033cb3b2dc456833057ce38715b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,041 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,048 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002 2024-11-24T01:46:18,077 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:18,083 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002 2024-11-24T01:46:18,087 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8aad8033cb3b2dc456833057ce38715b 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T01:46:18,134 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/.tmp/a/7b646a9ff3f142819371376e484e75c1 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732412777668/Put/seqid=0 2024-11-24T01:46:18,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741844_1020 (size=5170) 2024-11-24T01:46:18,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741844_1020 (size=5170) 2024-11-24T01:46:18,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741844_1020 (size=5170) 2024-11-24T01:46:18,147 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/.tmp/a/7b646a9ff3f142819371376e484e75c1 2024-11-24T01:46:18,189 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/.tmp/a/7b646a9ff3f142819371376e484e75c1 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/a/7b646a9ff3f142819371376e484e75c1 2024-11-24T01:46:18,198 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/a/7b646a9ff3f142819371376e484e75c1, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T01:46:18,203 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8aad8033cb3b2dc456833057ce38715b in 117ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T01:46:18,205 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/0000000000000000002 2024-11-24T01:46:18,206 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,206 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,209 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8aad8033cb3b2dc456833057ce38715b 2024-11-24T01:46:18,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/8aad8033cb3b2dc456833057ce38715b/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T01:46:18,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8aad8033cb3b2dc456833057ce38715b; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65706156, jitterRate=-0.020901978015899658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:18,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8aad8033cb3b2dc456833057ce38715b: Writing region info on filesystem at 1732412778036Initializing all the Stores at 1732412778037 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778037Obtaining lock to block concurrent updates at 1732412778087 (+50 ms)Preparing flush snapshotting stores in 8aad8033cb3b2dc456833057ce38715b at 1732412778087Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732412778089 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732412777553.8aad8033cb3b2dc456833057ce38715b. at 1732412778089Flushing 8aad8033cb3b2dc456833057ce38715b/a: creating writer at 1732412778090 (+1 ms)Flushing 8aad8033cb3b2dc456833057ce38715b/a: appending metadata at 1732412778123 (+33 ms)Flushing 8aad8033cb3b2dc456833057ce38715b/a: closing flushed file at 1732412778126 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39cbad59: reopening flushed file at 1732412778187 (+61 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8aad8033cb3b2dc456833057ce38715b in 117ms, sequenceid=2, compaction requested=false; wal=null at 1732412778203 (+16 ms)Cleaning up temporary data from old regions at 1732412778206 (+3 ms)Region opened successfully at 1732412778214 (+8 ms) 2024-11-24T01:46:18,238 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=379 (was 368) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39758 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:42186 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52866 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52702 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39612 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:42098 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:34675/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=7887 (was 7903) 2024-11-24T01:46:18,248 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=379, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=7886 2024-11-24T01:46:18,263 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:18,265 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:18,266 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:18,270 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37258856, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-37258856, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:18,282 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37258856/hregion-37258856.1732412778271, exclude list is [], retry=0 2024-11-24T01:46:18,286 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:18,286 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:18,286 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:18,313 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37258856/hregion-37258856.1732412778271 2024-11-24T01:46:18,313 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:18,313 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 45ccc4dc139afbceb6fb5ab32f59629a, NAME => 'testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:18,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741846_1022 (size=64) 2024-11-24T01:46:18,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741846_1022 (size=64) 2024-11-24T01:46:18,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741846_1022 (size=64) 2024-11-24T01:46:18,325 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:18,327 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,330 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ccc4dc139afbceb6fb5ab32f59629a columnFamilyName a 2024-11-24T01:46:18,330 DEBUG [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,331 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] regionserver.HStore(327): Store=45ccc4dc139afbceb6fb5ab32f59629a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,331 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,333 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,333 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,334 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,334 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,337 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,340 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:18,341 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45ccc4dc139afbceb6fb5ab32f59629a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74413697, jitterRate=0.10885049402713776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:18,341 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45ccc4dc139afbceb6fb5ab32f59629a: Writing region info on filesystem at 1732412778325Initializing all the Stores at 1732412778327 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778327Cleaning up temporary data from old regions at 1732412778334 (+7 ms)Region opened successfully at 1732412778341 (+7 ms) 2024-11-24T01:46:18,341 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 45ccc4dc139afbceb6fb5ab32f59629a, disabling compactions & flushes 2024-11-24T01:46:18,341 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. 2024-11-24T01:46:18,341 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. 2024-11-24T01:46:18,341 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. after waiting 0 ms 2024-11-24T01:46:18,341 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. 2024-11-24T01:46:18,342 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. 2024-11-24T01:46:18,342 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 45ccc4dc139afbceb6fb5ab32f59629a: Waiting for close lock at 1732412778341Disabling compacts and flushes for region at 1732412778341Disabling writes for close at 1732412778341Writing region close event to WAL at 1732412778342 (+1 ms)Closed at 1732412778342 2024-11-24T01:46:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741845_1021 (size=175) 2024-11-24T01:46:18,346 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-37258856/hregion-37258856.1732412778271 not finished, retry = 0 2024-11-24T01:46:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741845_1021 (size=175) 2024-11-24T01:46:18,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741845_1021 (size=175) 2024-11-24T01:46:18,450 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:18,450 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37258856:(num 1732412778271) 2024-11-24T01:46:18,451 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T01:46:18,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741847_1023 (size=469) 2024-11-24T01:46:18,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741847_1023 (size=469) 2024-11-24T01:46:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741847_1023 (size=469) 2024-11-24T01:46:18,490 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T01:46:18,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741848_1024 (size=322) 2024-11-24T01:46:18,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741848_1024 (size=322) 2024-11-24T01:46:18,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741848_1024 (size=322) 2024-11-24T01:46:18,543 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2, size=322 (322bytes) 2024-11-24T01:46:18,543 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2 2024-11-24T01:46:18,544 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2 after 1ms 2024-11-24T01:46:18,570 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:18,570 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2 took 28ms 2024-11-24T01:46:18,573 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2 so closing down 2024-11-24T01:46:18,573 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:18,575 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T01:46:18,599 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T01:46:18,599 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:18,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741849_1025 (size=322) 2024-11-24T01:46:18,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741849_1025 (size=322) 2024-11-24T01:46:18,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741849_1025 (size=322) 2024-11-24T01:46:18,608 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:18,610 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 2024-11-24T01:46:18,610 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T01:46:18,611 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2, journal: Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2, size=322 (322bytes) at 1732412778543Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2 so closing down at 1732412778573 (+30 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp at 1732412778599 (+26 ms)3 split writer threads finished at 1732412778599Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732412778608 (+9 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 at 1732412778610 (+2 ms)Processed 1 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732412778610 2024-11-24T01:46:18,631 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1, size=469 (469bytes) 2024-11-24T01:46:18,631 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1 2024-11-24T01:46:18,632 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1 after 1ms 2024-11-24T01:46:18,657 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:18,657 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1 took 27ms 2024-11-24T01:46:18,660 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1 so closing down 2024-11-24T01:46:18,660 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:18,662 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T01:46:18,687 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T01:46:18,688 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741850_1026 (size=469) 2024-11-24T01:46:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741850_1026 (size=469) 2024-11-24T01:46:18,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741850_1026 (size=469) 2024-11-24T01:46:18,696 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:18,722 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:18,724 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002, length=322 2024-11-24T01:46:18,726 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 2024-11-24T01:46:18,727 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T01:46:18,727 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1, journal: Splitting hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1, size=469 (469bytes) at 1732412778631Finishing writing output for hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1 so closing down at 1732412778660 (+29 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp at 1732412778687 (+27 ms)3 split writer threads finished at 1732412778688 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732412778696 (+8 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 at 1732412778727 (+31 ms)Processed 2 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732412778727 2024-11-24T01:46:18,727 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:18,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:18,742 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal.1732412778730, exclude list is [], retry=0 2024-11-24T01:46:18,745 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:18,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:18,746 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:18,770 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal.1732412778730 2024-11-24T01:46:18,771 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:18,771 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 45ccc4dc139afbceb6fb5ab32f59629a, NAME => 'testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:18,771 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:18,771 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,771 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,773 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,775 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45ccc4dc139afbceb6fb5ab32f59629a columnFamilyName a 2024-11-24T01:46:18,775 DEBUG [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,776 INFO [StoreOpener-45ccc4dc139afbceb6fb5ab32f59629a-1 {}] regionserver.HStore(327): Store=45ccc4dc139afbceb6fb5ab32f59629a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,776 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,777 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,779 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,780 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 2024-11-24T01:46:18,805 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:18,807 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 2024-11-24T01:46:18,807 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45ccc4dc139afbceb6fb5ab32f59629a 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T01:46:18,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/.tmp/a/b965d04cf2494222afe212ab690d9b23 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732412778450/Put/seqid=0 2024-11-24T01:46:18,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741852_1028 (size=5170) 2024-11-24T01:46:18,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741852_1028 (size=5170) 2024-11-24T01:46:18,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741852_1028 (size=5170) 2024-11-24T01:46:18,830 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/.tmp/a/b965d04cf2494222afe212ab690d9b23 2024-11-24T01:46:18,838 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/.tmp/a/b965d04cf2494222afe212ab690d9b23 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/a/b965d04cf2494222afe212ab690d9b23 2024-11-24T01:46:18,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/a/b965d04cf2494222afe212ab690d9b23, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T01:46:18,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 45ccc4dc139afbceb6fb5ab32f59629a in 39ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T01:46:18,847 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/0000000000000000002 2024-11-24T01:46:18,848 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,848 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,851 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 45ccc4dc139afbceb6fb5ab32f59629a 2024-11-24T01:46:18,853 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/45ccc4dc139afbceb6fb5ab32f59629a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T01:46:18,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 45ccc4dc139afbceb6fb5ab32f59629a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59459306, jitterRate=-0.11398729681968689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:18,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 45ccc4dc139afbceb6fb5ab32f59629a: Writing region info on filesystem at 1732412778771Initializing all the Stores at 1732412778773 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778773Obtaining lock to block concurrent updates at 1732412778807 (+34 ms)Preparing flush snapshotting stores in 45ccc4dc139afbceb6fb5ab32f59629a at 1732412778807Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732412778807Flushing stores of testReplayEditsWrittenIntoWAL,,1732412778264.45ccc4dc139afbceb6fb5ab32f59629a. at 1732412778807Flushing 45ccc4dc139afbceb6fb5ab32f59629a/a: creating writer at 1732412778807Flushing 45ccc4dc139afbceb6fb5ab32f59629a/a: appending metadata at 1732412778821 (+14 ms)Flushing 45ccc4dc139afbceb6fb5ab32f59629a/a: closing flushed file at 1732412778821Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@304e503c: reopening flushed file at 1732412778837 (+16 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 45ccc4dc139afbceb6fb5ab32f59629a in 39ms, sequenceid=2, compaction requested=false; wal=null at 1732412778847 (+10 ms)Cleaning up temporary data from old regions at 1732412778848 (+1 ms)Region opened successfully at 1732412778855 (+7 ms) 2024-11-24T01:46:18,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=389 (was 379) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52702 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52948 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:42098 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39854 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:42236 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=7873 (was 7886) 2024-11-24T01:46:18,881 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=389, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=7873 2024-11-24T01:46:18,897 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:18,899 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:18,900 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:18,903 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70782418, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-70782418, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:18,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70782418/hregion-70782418.1732412778904, exclude list is [], retry=0 2024-11-24T01:46:18,920 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:18,920 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:18,920 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:18,944 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70782418/hregion-70782418.1732412778904 2024-11-24T01:46:18,945 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:18,945 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ab84a9074aed0d92885cac1f226ff0c4, NAME => 'testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:18,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741854_1030 (size=64) 2024-11-24T01:46:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741854_1030 (size=64) 2024-11-24T01:46:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741854_1030 (size=64) 2024-11-24T01:46:18,963 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:18,964 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,966 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName a 2024-11-24T01:46:18,966 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,967 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,967 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,969 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName b 2024-11-24T01:46:18,969 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,969 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,970 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,971 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName c 2024-11-24T01:46:18,971 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:18,972 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:18,972 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,973 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,975 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,975 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,975 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:18,977 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:18,981 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:18,981 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab84a9074aed0d92885cac1f226ff0c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59290874, jitterRate=-0.11649712920188904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:18,982 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab84a9074aed0d92885cac1f226ff0c4: Writing region info on filesystem at 1732412778963Initializing all the Stores at 1732412778964 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778964Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778964Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412778964Cleaning up temporary data from old regions at 1732412778975 (+11 ms)Region opened successfully at 1732412778981 (+6 ms) 2024-11-24T01:46:18,982 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab84a9074aed0d92885cac1f226ff0c4, disabling compactions & flushes 2024-11-24T01:46:18,982 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:18,982 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:18,982 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. after waiting 0 ms 2024-11-24T01:46:18,982 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:18,983 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:18,983 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab84a9074aed0d92885cac1f226ff0c4: Waiting for close lock at 1732412778982Disabling compacts and flushes for region at 1732412778982Disabling writes for close at 1732412778982Writing region close event to WAL at 1732412778983 (+1 ms)Closed at 1732412778983 2024-11-24T01:46:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741853_1029 (size=175) 2024-11-24T01:46:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741853_1029 (size=175) 2024-11-24T01:46:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741853_1029 (size=175) 2024-11-24T01:46:18,990 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:18,990 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70782418:(num 1732412778904) 2024-11-24T01:46:18,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:18,992 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:19,004 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, exclude list is [], retry=0 2024-11-24T01:46:19,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:19,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:19,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:19,043 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 2024-11-24T01:46:19,043 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:19,237 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, size=0 (0bytes) 2024-11-24T01:46:19,237 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 might be still open, length is 0 2024-11-24T01:46:19,237 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 2024-11-24T01:46:19,238 WARN [IPC Server handler 3 on default port 34675 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T01:46:19,238 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 after 1ms 2024-11-24T01:46:22,034 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52968 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33251:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52968 dst: /127.0.0.1:33251 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33251 remote=/127.0.0.1:52968]. Total timeout mills is 60000, 57173 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:22,036 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39874 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39823:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39874 dst: /127.0.0.1:39823 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:22,036 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:42270 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42270 dst: /127.0.0.1:39381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:22,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741855_1032 (size=497775) 2024-11-24T01:46:22,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741855_1032 (size=497775) 2024-11-24T01:46:22,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741855_1032 (size=497775) 2024-11-24T01:46:22,672 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T01:46:22,727 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T01:46:23,239 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 after 4002ms 2024-11-24T01:46:23,274 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:23,275 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 took 4038ms 2024-11-24T01:46:23,280 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732412778993.temp 2024-11-24T01:46:23,320 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp 2024-11-24T01:46:23,391 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993; continuing. 2024-11-24T01:46:23,391 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 so closing down 2024-11-24T01:46:23,391 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:23,391 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:23,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741856_1033 (size=497783) 2024-11-24T01:46:23,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741856_1033 (size=497783) 2024-11-24T01:46:23,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741856_1033 (size=497783) 2024-11-24T01:46:23,394 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp (wrote 3002 edits, skipped 0 edits in 50 ms) 2024-11-24T01:46:23,396 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 2024-11-24T01:46:23,397 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T01:46:23,397 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, journal: Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, size=0 (0bytes) at 1732412779237Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp at 1732412783320 (+4083 ms)Split 1024 edits, skipped 0 edits. at 1732412783340 (+20 ms)Split 2048 edits, skipped 0 edits. at 1732412783368 (+28 ms)Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 so closing down at 1732412783391 (+23 ms)3 split writer threads finished at 1732412783391Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp (wrote 3002 edits, skipped 0 edits in 50 ms) at 1732412783395 (+4 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000000001-wal.1732412778993.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 at 1732412783396 (+1 ms)Processed 3002 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993, size=0, length=0, corrupted=false, cancelled=false at 1732412783397 (+1 ms) 2024-11-24T01:46:23,399 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412778993 2024-11-24T01:46:23,400 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 2024-11-24T01:46:23,401 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:23,403 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:23,418 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412783404, exclude list is [], retry=0 2024-11-24T01:46:23,421 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:23,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:23,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:23,449 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412783404 2024-11-24T01:46:23,449 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:23,450 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:23,452 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,453 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName a 2024-11-24T01:46:23,453 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:23,453 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:23,454 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,455 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName b 2024-11-24T01:46:23,455 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:23,455 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:23,455 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,457 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab84a9074aed0d92885cac1f226ff0c4 columnFamilyName c 2024-11-24T01:46:23,457 DEBUG [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:23,457 INFO [StoreOpener-ab84a9074aed0d92885cac1f226ff0c4-1 {}] regionserver.HStore(327): Store=ab84a9074aed0d92885cac1f226ff0c4/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:23,458 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,459 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,461 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:23,462 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 2024-11-24T01:46:23,489 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:23,527 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T01:46:23,862 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab84a9074aed0d92885cac1f226ff0c4 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T01:46:23,898 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/c1cffd647d924689a3102e96e8d78857 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732412779050/Put/seqid=0 2024-11-24T01:46:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741858_1035 (size=50463) 2024-11-24T01:46:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741858_1035 (size=50463) 2024-11-24T01:46:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741858_1035 (size=50463) 2024-11-24T01:46:23,907 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/c1cffd647d924689a3102e96e8d78857 2024-11-24T01:46:23,914 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/c1cffd647d924689a3102e96e8d78857 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/a/c1cffd647d924689a3102e96e8d78857 2024-11-24T01:46:23,921 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/a/c1cffd647d924689a3102e96e8d78857, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T01:46:23,922 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ab84a9074aed0d92885cac1f226ff0c4 in 60ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T01:46:23,944 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T01:46:23,944 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab84a9074aed0d92885cac1f226ff0c4 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T01:46:23,952 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/1ba3b8dfff634675b4bf9f11bf516fa3 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732412779089/Put/seqid=0 2024-11-24T01:46:23,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741859_1036 (size=20072) 2024-11-24T01:46:23,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741859_1036 (size=20072) 2024-11-24T01:46:23,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741859_1036 (size=20072) 2024-11-24T01:46:23,960 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/1ba3b8dfff634675b4bf9f11bf516fa3 2024-11-24T01:46:23,986 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/c296f430b2404f2a8ff6873194cb438c is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732412779118/Put/seqid=0 2024-11-24T01:46:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741860_1037 (size=35835) 2024-11-24T01:46:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741860_1037 (size=35835) 2024-11-24T01:46:23,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741860_1037 (size=35835) 2024-11-24T01:46:23,995 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/c296f430b2404f2a8ff6873194cb438c 2024-11-24T01:46:24,002 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/a/1ba3b8dfff634675b4bf9f11bf516fa3 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/a/1ba3b8dfff634675b4bf9f11bf516fa3 2024-11-24T01:46:24,009 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/a/1ba3b8dfff634675b4bf9f11bf516fa3, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T01:46:24,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/c296f430b2404f2a8ff6873194cb438c as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/b/c296f430b2404f2a8ff6873194cb438c 2024-11-24T01:46:24,018 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/b/c296f430b2404f2a8ff6873194cb438c, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T01:46:24,019 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ab84a9074aed0d92885cac1f226ff0c4 in 74ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T01:46:24,033 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T01:46:24,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab84a9074aed0d92885cac1f226ff0c4 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T01:46:24,040 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/aabae162f1f74798b0c31467ffeb46c6 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732412779135/Put/seqid=0 2024-11-24T01:46:24,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741861_1038 (size=35082) 2024-11-24T01:46:24,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741861_1038 (size=35082) 2024-11-24T01:46:24,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741861_1038 (size=35082) 2024-11-24T01:46:24,051 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/aabae162f1f74798b0c31467ffeb46c6 2024-11-24T01:46:24,082 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/e3b97c42baba49c696e1ecfde3080186 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732412779161/Put/seqid=0 2024-11-24T01:46:24,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741862_1039 (size=20825) 2024-11-24T01:46:24,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741862_1039 (size=20825) 2024-11-24T01:46:24,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741862_1039 (size=20825) 2024-11-24T01:46:24,091 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/e3b97c42baba49c696e1ecfde3080186 2024-11-24T01:46:24,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/b/aabae162f1f74798b0c31467ffeb46c6 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/b/aabae162f1f74798b0c31467ffeb46c6 2024-11-24T01:46:24,106 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/b/aabae162f1f74798b0c31467ffeb46c6, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T01:46:24,107 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/e3b97c42baba49c696e1ecfde3080186 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/c/e3b97c42baba49c696e1ecfde3080186 2024-11-24T01:46:24,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/c/e3b97c42baba49c696e1ecfde3080186, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T01:46:24,114 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ab84a9074aed0d92885cac1f226ff0c4 in 81ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T01:46:24,129 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732412779206/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:24,133 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 2024-11-24T01:46:24,134 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T01:46:24,134 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab84a9074aed0d92885cac1f226ff0c4 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T01:46:24,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/989dfb8d96ab4a9a9f2be2c74e7268b5 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732412779168/Put/seqid=0 2024-11-24T01:46:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741863_1040 (size=50301) 2024-11-24T01:46:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741863_1040 (size=50301) 2024-11-24T01:46:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741863_1040 (size=50301) 2024-11-24T01:46:24,158 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/989dfb8d96ab4a9a9f2be2c74e7268b5 2024-11-24T01:46:24,164 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 989dfb8d96ab4a9a9f2be2c74e7268b5 2024-11-24T01:46:24,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/.tmp/c/989dfb8d96ab4a9a9f2be2c74e7268b5 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/c/989dfb8d96ab4a9a9f2be2c74e7268b5 2024-11-24T01:46:24,173 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 989dfb8d96ab4a9a9f2be2c74e7268b5 2024-11-24T01:46:24,173 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/c/989dfb8d96ab4a9a9f2be2c74e7268b5, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T01:46:24,173 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ab84a9074aed0d92885cac1f226ff0c4 in 39ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T01:46:24,175 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/0000000000000003002 2024-11-24T01:46:24,176 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:24,176 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:24,177 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T01:46:24,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ab84a9074aed0d92885cac1f226ff0c4 2024-11-24T01:46:24,181 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenIntoWAL/ab84a9074aed0d92885cac1f226ff0c4/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T01:46:24,182 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ab84a9074aed0d92885cac1f226ff0c4; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66271736, jitterRate=-0.0124741792678833}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T01:46:24,182 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ab84a9074aed0d92885cac1f226ff0c4: Writing region info on filesystem at 1732412783450Initializing all the Stores at 1732412783451 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412783451Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412783452 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412783452Cleaning up temporary data from old regions at 1732412784176 (+724 ms)Region opened successfully at 1732412784182 (+6 ms) 2024-11-24T01:46:24,248 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ab84a9074aed0d92885cac1f226ff0c4, disabling compactions & flushes 2024-11-24T01:46:24,248 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:24,248 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:24,248 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. after waiting 0 ms 2024-11-24T01:46:24,248 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:24,250 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732412778897.ab84a9074aed0d92885cac1f226ff0c4. 2024-11-24T01:46:24,250 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ab84a9074aed0d92885cac1f226ff0c4: Waiting for close lock at 1732412784247Disabling compacts and flushes for region at 1732412784247Disabling writes for close at 1732412784248 (+1 ms)Writing region close event to WAL at 1732412784249 (+1 ms)Closed at 1732412784249 2024-11-24T01:46:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741857_1034 (size=175) 2024-11-24T01:46:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741857_1034 (size=175) 2024-11-24T01:46:24,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741857_1034 (size=175) 2024-11-24T01:46:24,257 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:24,257 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732412783404) 2024-11-24T01:46:24,275 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=403 (was 389) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:34675 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32809 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34675 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:32809 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2003590181_22 at /127.0.0.1:46458 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2003590181_22 at /127.0.0.1:48344 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35539 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2003590181_22 at /127.0.0.1:38892 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2003590181_22 at /127.0.0.1:38878 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:35539 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=7685 (was 7873) 2024-11-24T01:46:24,289 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=7681 2024-11-24T01:46:24,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T01:46:24,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:24,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T01:46:24,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T01:46:24,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T01:46:24,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:24,296 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T01:46:24,296 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:24,306 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:24,309 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:24,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:24,313 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-33856587, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-33856587, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:24,332 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-33856587/hregion-33856587.1732412784314, exclude list is [], retry=0 2024-11-24T01:46:24,336 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:24,336 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:24,336 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:24,375 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-33856587/hregion-33856587.1732412784314 2024-11-24T01:46:24,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:24,376 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 080b5126db1833d7f9f142a726290f0f, NAME => 'test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741865_1042 (size=43) 2024-11-24T01:46:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741865_1042 (size=43) 2024-11-24T01:46:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741865_1042 (size=43) 2024-11-24T01:46:24,387 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:24,389 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,391 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName a 2024-11-24T01:46:24,391 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:24,391 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:24,391 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,393 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName b 2024-11-24T01:46:24,393 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:24,394 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:24,394 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,396 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName c 2024-11-24T01:46:24,396 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:24,397 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:24,397 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,398 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,399 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,400 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,400 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,401 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:24,403 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:24,405 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:24,406 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 080b5126db1833d7f9f142a726290f0f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70722891, jitterRate=0.053853198885917664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:24,407 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 080b5126db1833d7f9f142a726290f0f: Writing region info on filesystem at 1732412784388Initializing all the Stores at 1732412784389 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412784389Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412784389Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412784389Cleaning up temporary data from old regions at 1732412784401 (+12 ms)Region opened successfully at 1732412784407 (+6 ms) 2024-11-24T01:46:24,407 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 080b5126db1833d7f9f142a726290f0f, disabling compactions & flushes 2024-11-24T01:46:24,407 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:24,407 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:24,407 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. after waiting 0 ms 2024-11-24T01:46:24,407 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:24,408 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:24,408 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 080b5126db1833d7f9f142a726290f0f: Waiting for close lock at 1732412784407Disabling compacts and flushes for region at 1732412784407Disabling writes for close at 1732412784407Writing region close event to WAL at 1732412784408 (+1 ms)Closed at 1732412784408 2024-11-24T01:46:24,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741864_1041 (size=175) 2024-11-24T01:46:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741864_1041 (size=175) 2024-11-24T01:46:24,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741864_1041 (size=175) 2024-11-24T01:46:24,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:24,415 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-33856587:(num 1732412784314) 2024-11-24T01:46:24,415 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:24,417 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:24,430 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, exclude list is [], retry=0 2024-11-24T01:46:24,433 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:24,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:24,434 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:24,463 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 2024-11-24T01:46:24,463 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:24,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741866_1043 (size=371388) 2024-11-24T01:46:24,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741866_1043 (size=371388) 2024-11-24T01:46:24,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741866_1043 (size=371388) 2024-11-24T01:46:24,648 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, size=362.7 K (371388bytes) 2024-11-24T01:46:24,648 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 2024-11-24T01:46:24,648 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 after 0ms 2024-11-24T01:46:24,683 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:24,684 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 took 37ms 2024-11-24T01:46:24,689 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732412784418.temp 2024-11-24T01:46:24,728 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 so closing down 2024-11-24T01:46:24,728 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:24,729 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp 2024-11-24T01:46:24,747 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:24,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741867_1044 (size=371388) 2024-11-24T01:46:24,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741867_1044 (size=371388) 2024-11-24T01:46:24,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741867_1044 (size=371388) 2024-11-24T01:46:24,752 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-11-24T01:46:24,753 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp to hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 2024-11-24T01:46:24,753 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-24T01:46:24,753 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, journal: Splitting hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, size=362.7 K (371388bytes) at 1732412784648Split 1024 edits, skipped 0 edits. at 1732412784699 (+51 ms)Split 2048 edits, skipped 0 edits. at 1732412784714 (+15 ms)Finishing writing output for hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 so closing down at 1732412784728 (+14 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp at 1732412784730 (+2 ms)3 split writer threads finished at 1732412784748 (+18 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1732412784752 (+4 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000000001-wal.1732412784418.temp to hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 at 1732412784753 (+1 ms)Processed 3000 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732412784753 2024-11-24T01:46:24,755 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784418 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412784418 2024-11-24T01:46:24,756 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 2024-11-24T01:46:24,757 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:24,759 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:24,774 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, exclude list is [], retry=0 2024-11-24T01:46:24,777 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:24,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:24,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:24,818 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 2024-11-24T01:46:24,819 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:25,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741868_1045 (size=371515) 2024-11-24T01:46:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741868_1045 (size=371515) 2024-11-24T01:46:25,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741868_1045 (size=371515) 2024-11-24T01:46:25,063 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, size=362.8 K (371515bytes) 2024-11-24T01:46:25,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 2024-11-24T01:46:25,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 after 0ms 2024-11-24T01:46:25,126 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:25,127 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 took 64ms 2024-11-24T01:46:25,131 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732412784759.temp 2024-11-24T01:46:25,156 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 so closing down 2024-11-24T01:46:25,156 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:25,166 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp 2024-11-24T01:46:25,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:25,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741869_1046 (size=371515) 2024-11-24T01:46:25,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741869_1046 (size=371515) 2024-11-24T01:46:25,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741869_1046 (size=371515) 2024-11-24T01:46:25,188 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-24T01:46:25,190 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp to hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 2024-11-24T01:46:25,190 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-24T01:46:25,190 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, journal: Splitting hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, size=362.8 K (371515bytes) at 1732412785063Split 1024 edits, skipped 0 edits. at 1732412785138 (+75 ms)Split 2048 edits, skipped 0 edits. at 1732412785147 (+9 ms)Finishing writing output for hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 so closing down at 1732412785156 (+9 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp at 1732412785166 (+10 ms)3 split writer threads finished at 1732412785177 (+11 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732412785188 (+11 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003001-wal.1732412784759.temp to hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 at 1732412785190 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732412785190 2024-11-24T01:46:25,192 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412784759 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412784759 2024-11-24T01:46:25,193 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 2024-11-24T01:46:25,194 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:25,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/test2727-manual,16010,1732412784306, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:25,216 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412785197, exclude list is [], retry=0 2024-11-24T01:46:25,219 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:25,220 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:25,220 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:25,246 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732412784306/wal.1732412785197 2024-11-24T01:46:25,247 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:25,247 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 080b5126db1833d7f9f142a726290f0f, NAME => 'test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:25,247 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:25,247 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,247 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,249 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,250 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName a 2024-11-24T01:46:25,250 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,250 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,251 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,252 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName b 2024-11-24T01:46:25,252 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,252 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,252 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,253 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080b5126db1833d7f9f142a726290f0f columnFamilyName c 2024-11-24T01:46:25,253 DEBUG [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,254 INFO [StoreOpener-080b5126db1833d7f9f142a726290f0f-1 {}] regionserver.HStore(327): Store=080b5126db1833d7f9f142a726290f0f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,254 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,254 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,257 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,258 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 2024-11-24T01:46:25,283 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:25,308 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 2024-11-24T01:46:25,310 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 2024-11-24T01:46:25,333 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:25,371 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 2024-11-24T01:46:25,372 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 080b5126db1833d7f9f142a726290f0f 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T01:46:25,400 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/a/f561a58255e946b9a8485468bc7e2b0a is 41, key is test2727/a:100/1732412784826/Put/seqid=0 2024-11-24T01:46:25,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741871_1048 (size=84227) 2024-11-24T01:46:25,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741871_1048 (size=84227) 2024-11-24T01:46:25,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741871_1048 (size=84227) 2024-11-24T01:46:25,406 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/a/f561a58255e946b9a8485468bc7e2b0a 2024-11-24T01:46:25,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/b/3495a781c57642aea5787825e5f17491 is 41, key is test2727/b:100/1732412784896/Put/seqid=0 2024-11-24T01:46:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741872_1049 (size=84609) 2024-11-24T01:46:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741872_1049 (size=84609) 2024-11-24T01:46:25,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741872_1049 (size=84609) 2024-11-24T01:46:25,444 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/b/3495a781c57642aea5787825e5f17491 2024-11-24T01:46:25,478 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/c/a32a5592c6db40a6af03b97809b30a06 is 41, key is test2727/c:100/1732412784968/Put/seqid=0 2024-11-24T01:46:25,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741873_1050 (size=84609) 2024-11-24T01:46:25,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741873_1050 (size=84609) 2024-11-24T01:46:25,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741873_1050 (size=84609) 2024-11-24T01:46:25,486 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/c/a32a5592c6db40a6af03b97809b30a06 2024-11-24T01:46:25,493 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/a/f561a58255e946b9a8485468bc7e2b0a as hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/a/f561a58255e946b9a8485468bc7e2b0a 2024-11-24T01:46:25,499 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/a/f561a58255e946b9a8485468bc7e2b0a, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T01:46:25,500 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/b/3495a781c57642aea5787825e5f17491 as hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/b/3495a781c57642aea5787825e5f17491 2024-11-24T01:46:25,506 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/b/3495a781c57642aea5787825e5f17491, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T01:46:25,508 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/.tmp/c/a32a5592c6db40a6af03b97809b30a06 as hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/c/a32a5592c6db40a6af03b97809b30a06 2024-11-24T01:46:25,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/c/a32a5592c6db40a6af03b97809b30a06, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T01:46:25,514 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 080b5126db1833d7f9f142a726290f0f in 142ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T01:46:25,515 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000003000 2024-11-24T01:46:25,516 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/0000000000000006000 2024-11-24T01:46:25,517 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,517 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,518 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:25,520 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 080b5126db1833d7f9f142a726290f0f 2024-11-24T01:46:25,523 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/test2727/080b5126db1833d7f9f142a726290f0f/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T01:46:25,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 080b5126db1833d7f9f142a726290f0f; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69347822, jitterRate=0.03336307406425476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:25,526 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 080b5126db1833d7f9f142a726290f0f: Writing region info on filesystem at 1732412785247Initializing all the Stores at 1732412785248 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785248Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785249 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785249Obtaining lock to block concurrent updates at 1732412785372 (+123 ms)Preparing flush snapshotting stores in 080b5126db1833d7f9f142a726290f0f at 1732412785372Finished memstore snapshotting test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732412785372Flushing stores of test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. at 1732412785372Flushing 080b5126db1833d7f9f142a726290f0f/a: creating writer at 1732412785372Flushing 080b5126db1833d7f9f142a726290f0f/a: appending metadata at 1732412785399 (+27 ms)Flushing 080b5126db1833d7f9f142a726290f0f/a: closing flushed file at 1732412785399Flushing 080b5126db1833d7f9f142a726290f0f/b: creating writer at 1732412785412 (+13 ms)Flushing 080b5126db1833d7f9f142a726290f0f/b: appending metadata at 1732412785437 (+25 ms)Flushing 080b5126db1833d7f9f142a726290f0f/b: closing flushed file at 1732412785437Flushing 080b5126db1833d7f9f142a726290f0f/c: creating writer at 1732412785451 (+14 ms)Flushing 080b5126db1833d7f9f142a726290f0f/c: appending metadata at 1732412785477 (+26 ms)Flushing 080b5126db1833d7f9f142a726290f0f/c: closing flushed file at 1732412785477Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622162b1: reopening flushed file at 1732412785491 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c508858: reopening flushed file at 1732412785499 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7be3c9e1: reopening flushed file at 1732412785507 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 080b5126db1833d7f9f142a726290f0f in 142ms, sequenceid=6000, compaction requested=false; wal=null at 1732412785514 (+7 ms)Cleaning up temporary data from old regions at 1732412785517 (+3 ms)Region opened successfully at 1732412785525 (+8 ms) 2024-11-24T01:46:25,527 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T01:46:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 080b5126db1833d7f9f142a726290f0f, disabling compactions & flushes 2024-11-24T01:46:25,527 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. after waiting 0 ms 2024-11-24T01:46:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:25,529 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732412784307.080b5126db1833d7f9f142a726290f0f. 2024-11-24T01:46:25,529 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 080b5126db1833d7f9f142a726290f0f: Waiting for close lock at 1732412785527Disabling compacts and flushes for region at 1732412785527Disabling writes for close at 1732412785527Writing region close event to WAL at 1732412785529 (+2 ms)Closed at 1732412785529 2024-11-24T01:46:25,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741870_1047 (size=175) 2024-11-24T01:46:25,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741870_1047 (size=175) 2024-11-24T01:46:25,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741870_1047 (size=175) 2024-11-24T01:46:25,534 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:25,534 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732412785197) 2024-11-24T01:46:25,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=408 (was 403) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:48458 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:38892 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:38878 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:46550 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=223 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7463 (was 7681) 2024-11-24T01:46:25,555 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=7463 2024-11-24T01:46:25,573 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:25,579 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:25,580 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732412785580 2024-11-24T01:46:25,609 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 2024-11-24T01:46:25,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:25,612 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a1eeb8e2d96f4196c418648fd180cf5b, NAME => 'testSequentialEditLogSeqNum,,1732412785573.a1eeb8e2d96f4196c418648fd180cf5b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:25,612 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732412785573.a1eeb8e2d96f4196c418648fd180cf5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:25,612 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,612 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,613 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b doesn't exist for region: a1eeb8e2d96f4196c418648fd180cf5b on table testSequentialEditLogSeqNum 2024-11-24T01:46:25,614 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: a1eeb8e2d96f4196c418648fd180cf5b on table testSequentialEditLogSeqNum 2024-11-24T01:46:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741875_1052 (size=62) 2024-11-24T01:46:25,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741875_1052 (size=62) 2024-11-24T01:46:25,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741875_1052 (size=62) 2024-11-24T01:46:25,625 INFO [StoreOpener-a1eeb8e2d96f4196c418648fd180cf5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,627 INFO [StoreOpener-a1eeb8e2d96f4196c418648fd180cf5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a1eeb8e2d96f4196c418648fd180cf5b columnFamilyName a 2024-11-24T01:46:25,627 DEBUG [StoreOpener-a1eeb8e2d96f4196c418648fd180cf5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,628 INFO [StoreOpener-a1eeb8e2d96f4196c418648fd180cf5b-1 {}] regionserver.HStore(327): Store=a1eeb8e2d96f4196c418648fd180cf5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,628 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,629 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,629 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,629 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,629 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,631 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a1eeb8e2d96f4196c418648fd180cf5b 2024-11-24T01:46:25,633 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:25,633 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a1eeb8e2d96f4196c418648fd180cf5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74600955, jitterRate=0.11164085566997528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:25,634 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a1eeb8e2d96f4196c418648fd180cf5b: Writing region info on filesystem at 1732412785612Initializing all the Stores at 1732412785624 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785624Cleaning up temporary data from old regions at 1732412785629 (+5 ms)Region opened successfully at 1732412785634 (+5 ms) 2024-11-24T01:46:25,645 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a1eeb8e2d96f4196c418648fd180cf5b 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T01:46:25,665 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/.tmp/a/9a483362806d4aef8aeb4eb91ff4e164 is 81, key is testSequentialEditLogSeqNum/a:x0/1732412785634/Put/seqid=0 2024-11-24T01:46:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741876_1053 (size=5833) 2024-11-24T01:46:25,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741876_1053 (size=5833) 2024-11-24T01:46:25,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741876_1053 (size=5833) 2024-11-24T01:46:25,676 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/.tmp/a/9a483362806d4aef8aeb4eb91ff4e164 2024-11-24T01:46:25,684 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/.tmp/a/9a483362806d4aef8aeb4eb91ff4e164 as hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/a/9a483362806d4aef8aeb4eb91ff4e164 2024-11-24T01:46:25,692 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/a/9a483362806d4aef8aeb4eb91ff4e164, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T01:46:25,693 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for a1eeb8e2d96f4196c418648fd180cf5b in 48ms, sequenceid=13, compaction requested=false 2024-11-24T01:46:25,693 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a1eeb8e2d96f4196c418648fd180cf5b: 2024-11-24T01:46:25,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T01:46:25,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T01:46:25,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T01:46:25,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T01:46:25,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T01:46:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741874_1051 (size=3256) 2024-11-24T01:46:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741874_1051 (size=3256) 2024-11-24T01:46:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741874_1051 (size=3256) 2024-11-24T01:46:25,724 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580, size=3.2 K (3256bytes) 2024-11-24T01:46:25,724 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 2024-11-24T01:46:25,725 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 after 1ms 2024-11-24T01:46:25,751 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:25,751 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 took 27ms 2024-11-24T01:46:25,754 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 so closing down 2024-11-24T01:46:25,755 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:25,756 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732412785580.temp 2024-11-24T01:46:25,783 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp 2024-11-24T01:46:25,783 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741877_1054 (size=2650) 2024-11-24T01:46:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741877_1054 (size=2650) 2024-11-24T01:46:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741877_1054 (size=2650) 2024-11-24T01:46:25,795 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:25,796 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp to hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000020 2024-11-24T01:46:25,797 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-24T01:46:25,797 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580, journal: Splitting hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580, size=3.2 K (3256bytes) at 1732412785724Finishing writing output for hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580 so closing down at 1732412785754 (+30 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp at 1732412785783 (+29 ms)3 split writer threads finished at 1732412785783Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732412785795 (+12 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000003-wal.1732412785580.temp to hdfs://localhost:34675/hbase/data/default/testSequentialEditLogSeqNum/a1eeb8e2d96f4196c418648fd180cf5b/recovered.edits/0000000000000000020 at 1732412785797 (+2 ms)Processed 17 edits across 1 Regions in 45 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732412785572/wal.1732412785580, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732412785797 2024-11-24T01:46:25,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=413 (was 408) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:46550 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=223 (was 223), ProcessCount=11 (was 11), AvailableMemoryMB=7457 (was 7463) 2024-11-24T01:46:25,822 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=223, ProcessCount=11, AvailableMemoryMB=7457 2024-11-24T01:46:25,836 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:25,838 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:25,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:25,866 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72878434, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-72878434, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:25,880 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72878434/hregion-72878434.1732412785866, exclude list is [], retry=0 2024-11-24T01:46:25,883 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:25,884 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:25,884 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:25,916 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72878434/hregion-72878434.1732412785866 2024-11-24T01:46:25,917 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:25,917 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5874e015dedfdbf51af1b60b04e030f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741879_1056 (size=70) 2024-11-24T01:46:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741879_1056 (size=70) 2024-11-24T01:46:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741879_1056 (size=70) 2024-11-24T01:46:25,927 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:25,928 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,930 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName a 2024-11-24T01:46:25,930 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,930 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,930 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,932 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName b 2024-11-24T01:46:25,932 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,932 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,933 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,934 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName c 2024-11-24T01:46:25,934 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:25,934 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:25,935 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,937 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,937 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,938 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:25,939 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,942 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:25,942 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5874e015dedfdbf51af1b60b04e030f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69050419, jitterRate=0.028931424021720886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:25,943 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5874e015dedfdbf51af1b60b04e030f8: Writing region info on filesystem at 1732412785927Initializing all the Stores at 1732412785928 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785928Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785928Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785928Cleaning up temporary data from old regions at 1732412785937 (+9 ms)Region opened successfully at 1732412785943 (+6 ms) 2024-11-24T01:46:25,944 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5874e015dedfdbf51af1b60b04e030f8, disabling compactions & flushes 2024-11-24T01:46:25,944 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:25,944 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:25,944 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. after waiting 0 ms 2024-11-24T01:46:25,944 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:25,944 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:25,944 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5874e015dedfdbf51af1b60b04e030f8: Waiting for close lock at 1732412785943Disabling compacts and flushes for region at 1732412785943Disabling writes for close at 1732412785944 (+1 ms)Writing region close event to WAL at 1732412785944Closed at 1732412785944 2024-11-24T01:46:25,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741878_1055 (size=175) 2024-11-24T01:46:25,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741878_1055 (size=175) 2024-11-24T01:46:25,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741878_1055 (size=175) 2024-11-24T01:46:25,950 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:25,951 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72878434:(num 1732412785866) 2024-11-24T01:46:25,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:25,953 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:25,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, exclude list is [], retry=0 2024-11-24T01:46:25,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:25,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:25,969 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:25,995 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 2024-11-24T01:46:25,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:25,996 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5874e015dedfdbf51af1b60b04e030f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:25,997 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:25,997 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,997 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:25,999 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,000 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName a 2024-11-24T01:46:26,000 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:26,000 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:26,001 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,002 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName b 2024-11-24T01:46:26,002 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:26,002 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:26,002 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,003 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName c 2024-11-24T01:46:26,003 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:26,004 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:26,004 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,005 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,006 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,007 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,007 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,008 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:26,009 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:26,010 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5874e015dedfdbf51af1b60b04e030f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59618366, jitterRate=-0.11161711812019348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:26,011 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5874e015dedfdbf51af1b60b04e030f8: Writing region info on filesystem at 1732412785997Initializing all the Stores at 1732412785998 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785998Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785998Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412785998Cleaning up temporary data from old regions at 1732412786007 (+9 ms)Region opened successfully at 1732412786011 (+4 ms) 2024-11-24T01:46:26,016 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732412786015/Put/seqid=0 2024-11-24T01:46:26,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741881_1058 (size=4826) 2024-11-24T01:46:26,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741881_1058 (size=4826) 2024-11-24T01:46:26,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741881_1058 (size=4826) 2024-11-24T01:46:26,025 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34675/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 5874e015dedfdbf51af1b60b04e030f8/a 2024-11-24T01:46:26,033 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T01:46:26,033 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T01:46:26,033 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5874e015dedfdbf51af1b60b04e030f8: 2024-11-24T01:46:26,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/26ef115556bb401fbb7b81c08027f38a_SeqId_3_ 2024-11-24T01:46:26,035 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34675/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 5874e015dedfdbf51af1b60b04e030f8/a as hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/26ef115556bb401fbb7b81c08027f38a_SeqId_3_ - updating store file list. 2024-11-24T01:46:26,041 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 26ef115556bb401fbb7b81c08027f38a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:26,041 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/26ef115556bb401fbb7b81c08027f38a_SeqId_3_ into 5874e015dedfdbf51af1b60b04e030f8/a 2024-11-24T01:46:26,041 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34675/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 5874e015dedfdbf51af1b60b04e030f8/a (new location: hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/26ef115556bb401fbb7b81c08027f38a_SeqId_3_) 2024-11-24T01:46:26,090 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, size=0 (0bytes) 2024-11-24T01:46:26,090 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 might be still open, length is 0 2024-11-24T01:46:26,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 2024-11-24T01:46:26,090 WARN [IPC Server handler 0 on default port 34675 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T01:46:26,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 after 0ms 2024-11-24T01:46:28,018 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:48544 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48544 dst: /127.0.0.1:39381 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39381 remote=/127.0.0.1:48544]. Total timeout mills is 60000, 58029 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:28,019 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:46646 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39823:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46646 dst: /127.0.0.1:39823 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:28,019 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39124 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33251:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39124 dst: /127.0.0.1:33251 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:28,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741880_1059 (size=606) 2024-11-24T01:46:28,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741880_1059 (size=606) 2024-11-24T01:46:30,093 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 after 4002ms 2024-11-24T01:46:30,124 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:30,125 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 took 4035ms 2024-11-24T01:46:30,127 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954; continuing. 2024-11-24T01:46:30,127 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 so closing down 2024-11-24T01:46:30,127 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:30,128 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732412785954.temp 2024-11-24T01:46:30,157 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp 2024-11-24T01:46:30,158 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:30,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741882_1060 (size=329) 2024-11-24T01:46:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741882_1060 (size=329) 2024-11-24T01:46:30,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741882_1060 (size=329) 2024-11-24T01:46:30,164 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:30,165 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp to hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 2024-11-24T01:46:30,166 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T01:46:30,166 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, journal: Splitting hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, size=0 (0bytes) at 1732412786090Finishing writing output for hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 so closing down at 1732412790127 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp at 1732412790158 (+31 ms)3 split writer threads finished at 1732412790158Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732412790164 (+6 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005-wal.1732412785954.temp to hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 at 1732412790165 (+1 ms)Processed 2 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954, size=0, length=0, corrupted=false, cancelled=false at 1732412790166 (+1 ms) 2024-11-24T01:46:30,168 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412785954 2024-11-24T01:46:30,168 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 2024-11-24T01:46:30,169 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:30,170 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:30,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412790171, exclude list is [], retry=0 2024-11-24T01:46:30,192 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:30,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:30,193 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:30,221 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412790171 2024-11-24T01:46:30,222 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:30,222 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5874e015dedfdbf51af1b60b04e030f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:30,222 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:30,222 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,222 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,224 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,225 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName a 2024-11-24T01:46:30,225 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,231 DEBUG [StoreFileOpener-5874e015dedfdbf51af1b60b04e030f8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 26ef115556bb401fbb7b81c08027f38a_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:30,232 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/26ef115556bb401fbb7b81c08027f38a_SeqId_3_ 2024-11-24T01:46:30,232 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:30,232 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,233 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName b 2024-11-24T01:46:30,233 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,233 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:30,234 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,234 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5874e015dedfdbf51af1b60b04e030f8 columnFamilyName c 2024-11-24T01:46:30,234 DEBUG [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,235 INFO [StoreOpener-5874e015dedfdbf51af1b60b04e030f8-1 {}] regionserver.HStore(327): Store=5874e015dedfdbf51af1b60b04e030f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:30,235 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,239 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 2024-11-24T01:46:30,265 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:30,266 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 2024-11-24T01:46:30,266 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5874e015dedfdbf51af1b60b04e030f8 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T01:46:30,279 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/.tmp/a/0f687ae4a6054c989d7eda82744cc6de is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732412786046/Put/seqid=0 2024-11-24T01:46:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741884_1062 (size=5149) 2024-11-24T01:46:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741884_1062 (size=5149) 2024-11-24T01:46:30,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741884_1062 (size=5149) 2024-11-24T01:46:30,286 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/.tmp/a/0f687ae4a6054c989d7eda82744cc6de 2024-11-24T01:46:30,292 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/.tmp/a/0f687ae4a6054c989d7eda82744cc6de as hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/0f687ae4a6054c989d7eda82744cc6de 2024-11-24T01:46:30,298 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/a/0f687ae4a6054c989d7eda82744cc6de, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T01:46:30,299 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 5874e015dedfdbf51af1b60b04e030f8 in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T01:46:30,299 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/0000000000000000005 2024-11-24T01:46:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,301 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,301 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:30,303 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5874e015dedfdbf51af1b60b04e030f8 2024-11-24T01:46:30,305 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/5874e015dedfdbf51af1b60b04e030f8/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T01:46:30,306 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5874e015dedfdbf51af1b60b04e030f8; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64186759, jitterRate=-0.043542757630348206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:30,306 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5874e015dedfdbf51af1b60b04e030f8: Writing region info on filesystem at 1732412790222Initializing all the Stores at 1732412790224 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412790224Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412790224Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412790224Obtaining lock to block concurrent updates at 1732412790266 (+42 ms)Preparing flush snapshotting stores in 5874e015dedfdbf51af1b60b04e030f8 at 1732412790266Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732412790266Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. at 1732412790266Flushing 5874e015dedfdbf51af1b60b04e030f8/a: creating writer at 1732412790266Flushing 5874e015dedfdbf51af1b60b04e030f8/a: appending metadata at 1732412790279 (+13 ms)Flushing 5874e015dedfdbf51af1b60b04e030f8/a: closing flushed file at 1732412790279Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ef26f3f: reopening flushed file at 1732412790291 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 5874e015dedfdbf51af1b60b04e030f8 in 33ms, sequenceid=5, compaction requested=false; wal=null at 1732412790299 (+8 ms)Cleaning up temporary data from old regions at 1732412790301 (+2 ms)Region opened successfully at 1732412790306 (+5 ms) 2024-11-24T01:46:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5874e015dedfdbf51af1b60b04e030f8, disabling compactions & flushes 2024-11-24T01:46:30,310 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. after waiting 0 ms 2024-11-24T01:46:30,310 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:30,311 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732412785836.5874e015dedfdbf51af1b60b04e030f8. 2024-11-24T01:46:30,311 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5874e015dedfdbf51af1b60b04e030f8: Waiting for close lock at 1732412790310Disabling compacts and flushes for region at 1732412790310Disabling writes for close at 1732412790310Writing region close event to WAL at 1732412790311 (+1 ms)Closed at 1732412790311 2024-11-24T01:46:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741883_1061 (size=175) 2024-11-24T01:46:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741883_1061 (size=175) 2024-11-24T01:46:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741883_1061 (size=175) 2024-11-24T01:46:30,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:30,316 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732412790171) 2024-11-24T01:46:30,327 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=416 (was 413) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1784066768_22 at /127.0.0.1:46668 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1784066768_22 at /127.0.0.1:39128 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34675 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1784066768_22 at /127.0.0.1:48584 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:34675 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 223) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7465 (was 7457) - AvailableMemoryMB LEAK? - 2024-11-24T01:46:30,328 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-24T01:46:30,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=416, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=7464 2024-11-24T01:46:30,338 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-24T01:46:30,355 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:30,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T01:46:30,362 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a67e7d17e593,37581,1732412773942 2024-11-24T01:46:30,364 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@56b394a1 2024-11-24T01:46:30,364 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T01:46:30,366 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T01:46:30,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T01:46:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T01:46:30,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T01:46:30,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T01:46:30,381 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T01:46:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T01:46:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741885_1063 (size=694) 2024-11-24T01:46:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741885_1063 (size=694) 2024-11-24T01:46:30,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741885_1063 (size=694) 2024-11-24T01:46:30,395 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352 2024-11-24T01:46:30,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741886_1064 (size=77) 2024-11-24T01:46:30,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741886_1064 (size=77) 2024-11-24T01:46:30,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741886_1064 (size=77) 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:30,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412790406Disabling compacts and flushes for region at 1732412790406Disabling writes for close at 1732412790406Writing region close event to WAL at 1732412790406Closed at 1732412790406 2024-11-24T01:46:30,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T01:46:30,412 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732412790408"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732412790408"}]},"ts":"1732412790408"} 2024-11-24T01:46:30,415 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T01:46:30,416 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T01:46:30,419 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732412790416"}]},"ts":"1732412790416"} 2024-11-24T01:46:30,422 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T01:46:30,423 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a67e7d17e593=0} racks are {/default-rack=0} 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T01:46:30,424 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T01:46:30,424 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T01:46:30,424 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T01:46:30,424 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T01:46:30,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN}] 2024-11-24T01:46:30,428 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN 2024-11-24T01:46:30,429 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN; state=OFFLINE, location=a67e7d17e593,40811,1732412774716; forceNewPlan=false, retain=false 2024-11-24T01:46:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T01:46:30,581 INFO [a67e7d17e593:37581 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T01:46:30,582 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPENING, regionLocation=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:30,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN because future has completed 2024-11-24T01:46:30,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716}] 2024-11-24T01:46:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T01:46:30,740 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T01:46:30,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42677, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T01:46:30,746 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,747 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:30,747 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,747 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:30,747 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,747 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,749 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,751 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:30,751 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,751 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:30,752 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,753 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:30,753 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:30,754 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:30,754 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,755 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,756 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,757 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,757 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,758 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T01:46:30,759 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,762 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:30,763 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5bd22b2d59c54a42edcaf121e896c37c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69109983, jitterRate=0.029818996787071228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T01:46:30,763 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:30,764 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5bd22b2d59c54a42edcaf121e896c37c: Running coprocessor pre-open hook at 1732412790747Writing region info on filesystem at 1732412790747Initializing all the Stores at 1732412790748 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412790748Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412790749 (+1 ms)Cleaning up temporary data from old regions at 1732412790757 (+8 ms)Running coprocessor post-open hooks at 1732412790763 (+6 ms)Region opened successfully at 1732412790764 (+1 ms) 2024-11-24T01:46:30,766 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., pid=6, masterSystemTime=1732412790740 2024-11-24T01:46:30,768 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,768 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:30,770 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPEN, openSeqNum=2, regionLocation=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:30,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 because future has completed 2024-11-24T01:46:30,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T01:46:30,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 in 189 msec 2024-11-24T01:46:30,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T01:46:30,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN in 352 msec 2024-11-24T01:46:30,783 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T01:46:30,783 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732412790783"}]},"ts":"1732412790783"} 2024-11-24T01:46:30,785 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T01:46:30,787 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T01:46:30,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 416 msec 2024-11-24T01:46:31,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T01:46:31,016 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T01:46:31,016 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T01:46:31,017 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T01:46:31,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T01:46:31,023 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T01:46:31,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T01:46:31,036 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=2] 2024-11-24T01:46:31,037 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T01:46:31,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T01:46:31,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5bd22b2d59c54a42edcaf121e896c37c, source=a67e7d17e593,40811,1732412774716, destination=a67e7d17e593,33283,1732412774831, warming up region on a67e7d17e593,33283,1732412774831 2024-11-24T01:46:31,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5bd22b2d59c54a42edcaf121e896c37c, source=a67e7d17e593,40811,1732412774716, destination=a67e7d17e593,33283,1732412774831, running balancer 2024-11-24T01:46:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE 2024-11-24T01:46:31,059 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE 2024-11-24T01:46:31,061 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=CLOSING, regionLocation=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:31,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:31,062 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,064 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:31,064 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:31,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE because future has completed 2024-11-24T01:46:31,064 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T01:46:31,064 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:31,065 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716}] 2024-11-24T01:46:31,065 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:31,066 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:31,066 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:31,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,067 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412791066Disabling compacts and flushes for region at 1732412791066Disabling writes for close at 1732412791067 (+1 ms)Writing region close event to WAL at 1732412791067Closed at 1732412791067 2024-11-24T01:46:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T01:46:31,223 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,223 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T01:46:31,224 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:31,224 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,224 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,224 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:31,224 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,224 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 5bd22b2d59c54a42edcaf121e896c37c 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T01:46:31,241 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/cb8795c6c65d4a31944a31c58d1e946f is 35, key is r1/cf1:q/1732412791040/Put/seqid=0 2024-11-24T01:46:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741887_1065 (size=4783) 2024-11-24T01:46:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741887_1065 (size=4783) 2024-11-24T01:46:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741887_1065 (size=4783) 2024-11-24T01:46:31,248 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/cb8795c6c65d4a31944a31c58d1e946f 2024-11-24T01:46:31,257 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/cb8795c6c65d4a31944a31c58d1e946f as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f 2024-11-24T01:46:31,264 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T01:46:31,265 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5bd22b2d59c54a42edcaf121e896c37c in 41ms, sequenceid=5, compaction requested=false 2024-11-24T01:46:31,266 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T01:46:31,271 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T01:46:31,274 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,274 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412791224Running coprocessor pre-close hooks at 1732412791224Disabling compacts and flushes for region at 1732412791224Disabling writes for close at 1732412791224Obtaining lock to block concurrent updates at 1732412791224Preparing flush snapshotting stores in 5bd22b2d59c54a42edcaf121e896c37c at 1732412791224Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732412791225 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. at 1732412791225Flushing 5bd22b2d59c54a42edcaf121e896c37c/cf1: creating writer at 1732412791226 (+1 ms)Flushing 5bd22b2d59c54a42edcaf121e896c37c/cf1: appending metadata at 1732412791241 (+15 ms)Flushing 5bd22b2d59c54a42edcaf121e896c37c/cf1: closing flushed file at 1732412791241Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52be3394: reopening flushed file at 1732412791256 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 5bd22b2d59c54a42edcaf121e896c37c in 41ms, sequenceid=5, compaction requested=false at 1732412791266 (+10 ms)Writing region close event to WAL at 1732412791267 (+1 ms)Running coprocessor post-close hooks at 1732412791272 (+5 ms)Closed at 1732412791273 (+1 ms) 2024-11-24T01:46:31,274 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 5bd22b2d59c54a42edcaf121e896c37c move to a67e7d17e593,33283,1732412774831 record at close sequenceid=5 2024-11-24T01:46:31,277 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=CLOSED 2024-11-24T01:46:31,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 because future has completed 2024-11-24T01:46:31,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T01:46:31,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 in 270 msec 2024-11-24T01:46:31,341 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,33283,1732412774831; forceNewPlan=false, retain=false 2024-11-24T01:46:31,491 INFO [a67e7d17e593:37581 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T01:46:31,492 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPENING, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:31,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE because future has completed 2024-11-24T01:46:31,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831}] 2024-11-24T01:46:31,651 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,652 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:31,652 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,652 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:31,652 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,652 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,654 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,655 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:31,655 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:31,661 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f 2024-11-24T01:46:31,661 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:31,661 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,662 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:31,662 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:31,663 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:31,663 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,664 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,665 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,666 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,666 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,667 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T01:46:31,669 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,670 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 5bd22b2d59c54a42edcaf121e896c37c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65052718, jitterRate=-0.030638962984085083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T01:46:31,670 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:31,670 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 5bd22b2d59c54a42edcaf121e896c37c: Running coprocessor pre-open hook at 1732412791652Writing region info on filesystem at 1732412791652Initializing all the Stores at 1732412791653 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412791653Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412791654 (+1 ms)Cleaning up temporary data from old regions at 1732412791666 (+12 ms)Running coprocessor post-open hooks at 1732412791670 (+4 ms)Region opened successfully at 1732412791670 2024-11-24T01:46:31,672 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., pid=9, masterSystemTime=1732412791647 2024-11-24T01:46:31,674 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,674 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPEN, openSeqNum=9, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:31,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 because future has completed 2024-11-24T01:46:31,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T01:46:31,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 in 184 msec 2024-11-24T01:46:31,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE in 625 msec 2024-11-24T01:46:31,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T01:46:31,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T01:46:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:44836 deadline: 1732412851724, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=33283 startCode=1732412774831. As of locationSeqNum=5. 2024-11-24T01:46:31,731 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=33283 startCode=1732412774831. As of locationSeqNum=5. 2024-11-24T01:46:31,732 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=33283 startCode=1732412774831. As of locationSeqNum=5. 2024-11-24T01:46:31,732 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=33283 startCode=1732412774831. As of locationSeqNum=5. 2024-11-24T01:46:31,860 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5bd22b2d59c54a42edcaf121e896c37c 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T01:46:31,876 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/7956b65a543a45f09f22ab63458b6cb8 is 29, key is r1/cf1:/1732412791847/DeleteFamily/seqid=0 2024-11-24T01:46:31,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741888_1066 (size=4906) 2024-11-24T01:46:31,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741888_1066 (size=4906) 2024-11-24T01:46:31,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741888_1066 (size=4906) 2024-11-24T01:46:31,936 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:31,944 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:31,959 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf2/01b550a89c8a4533bd9a83c366fcd663 is 29, key is r1/cf2:/1732412791847/DeleteFamily/seqid=0 2024-11-24T01:46:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741889_1067 (size=4906) 2024-11-24T01:46:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741889_1067 (size=4906) 2024-11-24T01:46:31,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741889_1067 (size=4906) 2024-11-24T01:46:31,966 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf2/01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:31,971 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:31,972 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/7956b65a543a45f09f22ab63458b6cb8 as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:31,979 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:31,979 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T01:46:31,980 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf2/01b550a89c8a4533bd9a83c366fcd663 as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:31,987 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:31,987 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T01:46:31,988 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 5bd22b2d59c54a42edcaf121e896c37c in 128ms, sequenceid=12, compaction requested=false 2024-11-24T01:46:31,989 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5bd22b2d59c54a42edcaf121e896c37c: 2024-11-24T01:46:31,991 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T01:46:31,992 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5bd22b2d59c54a42edcaf121e896c37c/cf1 is initiating major compaction (all files) 2024-11-24T01:46:31,992 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T01:46:31,992 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:31,993 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5bd22b2d59c54a42edcaf121e896c37c/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:31,993 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f, hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8] into tmpdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp, totalSize=9.5 K 2024-11-24T01:46:31,994 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb8795c6c65d4a31944a31c58d1e946f, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732412791040 2024-11-24T01:46:31,995 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7956b65a543a45f09f22ab63458b6cb8, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T01:46:32,006 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5bd22b2d59c54a42edcaf121e896c37c#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T01:46:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741890_1068 (size=4626) 2024-11-24T01:46:32,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741890_1068 (size=4626) 2024-11-24T01:46:32,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741890_1068 (size=4626) 2024-11-24T01:46:32,020 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf1/d18a2e08aeef47bb85416f7f779a7159 as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/d18a2e08aeef47bb85416f7f779a7159 2024-11-24T01:46:32,035 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 5bd22b2d59c54a42edcaf121e896c37c/cf1 of 5bd22b2d59c54a42edcaf121e896c37c into d18a2e08aeef47bb85416f7f779a7159(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T01:46:32,035 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5bd22b2d59c54a42edcaf121e896c37c: 2024-11-24T01:46:32,035 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T01:46:32,035 DEBUG [Time-limited test {}] regionserver.HStore(1541): 5bd22b2d59c54a42edcaf121e896c37c/cf2 is initiating major compaction (all files) 2024-11-24T01:46:32,035 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T01:46:32,035 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T01:46:32,035 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 5bd22b2d59c54a42edcaf121e896c37c/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,036 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663] into tmpdir=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp, totalSize=4.8 K 2024-11-24T01:46:32,036 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 01b550a89c8a4533bd9a83c366fcd663, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T01:46:32,042 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 5bd22b2d59c54a42edcaf121e896c37c#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T01:46:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741891_1069 (size=4592) 2024-11-24T01:46:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741891_1069 (size=4592) 2024-11-24T01:46:32,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741891_1069 (size=4592) 2024-11-24T01:46:32,056 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/.tmp/cf2/6a09724ed7b54a8da3aafdd364b12237 as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/6a09724ed7b54a8da3aafdd364b12237 2024-11-24T01:46:32,064 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 5bd22b2d59c54a42edcaf121e896c37c/cf2 of 5bd22b2d59c54a42edcaf121e896c37c into 6a09724ed7b54a8da3aafdd364b12237(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T01:46:32,064 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 5bd22b2d59c54a42edcaf121e896c37c: 2024-11-24T01:46:32,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=5bd22b2d59c54a42edcaf121e896c37c, source=a67e7d17e593,33283,1732412774831, destination=a67e7d17e593,40811,1732412774716, warming up region on a67e7d17e593,40811,1732412774716 2024-11-24T01:46:32,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=5bd22b2d59c54a42edcaf121e896c37c, source=a67e7d17e593,33283,1732412774831, destination=a67e7d17e593,40811,1732412774716, running balancer 2024-11-24T01:46:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE 2024-11-24T01:46:32,071 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE 2024-11-24T01:46:32,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=CLOSING, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:32,075 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(7855): Warmup {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:32,076 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,077 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:32,077 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:32,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE because future has completed 2024-11-24T01:46:32,078 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T01:46:32,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831}] 2024-11-24T01:46:32,087 INFO [StoreFileOpener-5bd22b2d59c54a42edcaf121e896c37c-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:32,087 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:32,092 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f 2024-11-24T01:46:32,098 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/d18a2e08aeef47bb85416f7f779a7159 2024-11-24T01:46:32,098 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:32,098 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,099 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:32,099 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:32,106 INFO [StoreFileOpener-5bd22b2d59c54a42edcaf121e896c37c-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:32,106 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:32,111 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/6a09724ed7b54a8da3aafdd364b12237 2024-11-24T01:46:32,112 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:32,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40811 {}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412792112Disabling compacts and flushes for region at 1732412792112Disabling writes for close at 1732412792112Writing region close event to WAL at 1732412792113 (+1 ms)Closed at 1732412792113 2024-11-24T01:46:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T01:46:32,232 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,233 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T01:46:32,233 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:32,233 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,233 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,233 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:32,233 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,233 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f, hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8] to archive 2024-11-24T01:46:32,236 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T01:46:32,239 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/cb8795c6c65d4a31944a31c58d1e946f 2024-11-24T01:46:32,241 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8 to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/7956b65a543a45f09f22ab63458b6cb8 2024-11-24T01:46:32,254 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663] to archive 2024-11-24T01:46:32,255 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T01:46:32,257 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663 to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/01b550a89c8a4533bd9a83c366fcd663 2024-11-24T01:46:32,263 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T01:46:32,263 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,263 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412792233Running coprocessor pre-close hooks at 1732412792233Disabling compacts and flushes for region at 1732412792233Disabling writes for close at 1732412792233Writing region close event to WAL at 1732412792259 (+26 ms)Running coprocessor post-close hooks at 1732412792263 (+4 ms)Closed at 1732412792263 2024-11-24T01:46:32,263 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 5bd22b2d59c54a42edcaf121e896c37c move to a67e7d17e593,40811,1732412774716 record at close sequenceid=12 2024-11-24T01:46:32,266 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,267 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=CLOSED 2024-11-24T01:46:32,269 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 because future has completed 2024-11-24T01:46:32,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T01:46:32,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 in 192 msec 2024-11-24T01:46:32,274 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE; state=CLOSED, location=a67e7d17e593,40811,1732412774716; forceNewPlan=false, retain=false 2024-11-24T01:46:32,424 INFO [a67e7d17e593:37581 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T01:46:32,425 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPENING, regionLocation=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:32,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE because future has completed 2024-11-24T01:46:32,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716}] 2024-11-24T01:46:32,591 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,591 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:32,592 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,592 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:32,592 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,592 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,593 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,594 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:32,595 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:32,601 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/d18a2e08aeef47bb85416f7f779a7159 2024-11-24T01:46:32,601 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:32,602 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,603 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:32,603 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:32,610 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/6a09724ed7b54a8da3aafdd364b12237 2024-11-24T01:46:32,610 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:32,610 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,611 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,612 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,613 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,613 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,614 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T01:46:32,615 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,616 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 5bd22b2d59c54a42edcaf121e896c37c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71053661, jitterRate=0.05878205597400665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T01:46:32,616 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,617 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 5bd22b2d59c54a42edcaf121e896c37c: Running coprocessor pre-open hook at 1732412792592Writing region info on filesystem at 1732412792592Initializing all the Stores at 1732412792593 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412792593Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412792593Cleaning up temporary data from old regions at 1732412792613 (+20 ms)Running coprocessor post-open hooks at 1732412792616 (+3 ms)Region opened successfully at 1732412792617 (+1 ms) 2024-11-24T01:46:32,618 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., pid=12, masterSystemTime=1732412792585 2024-11-24T01:46:32,620 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,620 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,621 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPEN, openSeqNum=18, regionLocation=a67e7d17e593,40811,1732412774716 2024-11-24T01:46:32,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 because future has completed 2024-11-24T01:46:32,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T01:46:32,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,40811,1732412774716 in 193 msec 2024-11-24T01:46:32,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, REOPEN/MOVE in 557 msec 2024-11-24T01:46:32,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T01:46:32,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T01:46:32,676 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a67e7d17e593,40811,1732412774716: testing ***** 2024-11-24T01:46:32,676 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T01:46:32,678 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T01:46:32,679 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T01:46:32,683 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T01:46:32,684 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T01:46:32,693 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 488838144 }, "NonHeapMemoryUsage": { "committed": 171966464, "init": 7667712, "max": -1, "used": 169412672 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a67e7d17e593", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 32, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 8, "ProcessCallTime_median": 16, "ProcessCallTime_75th_percentile": 24, "ProcessCallTime_90th_percentile": 28, "ProcessCallTime_95th_percentile": 30, "ProcessCallTime_98th_percentile": 31, "ProcessCallTime_99th_percentile": 31, "ProcessCallTime_99.9th_percentile": 31, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 32, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 8, "TotalCallTime_median": 16, "TotalCallTime_75th_percentile": 24, "TotalCallTime_90th_percentile": 28, "TotalCallTime_95th_percentile": 30, "TotalCallTime_98th_percentile": 31, "TotalCallTime_99th_percentile": 31, "TotalCallTime_99.9th_percentile": 31, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a67e7d17e593", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60515", "tag.serverName": "a67e7d17e593,43635,1732412774891", "tag.clusterId": "7e44e35e-5659-455a-a56b-5baa5947e9b1", "tag.Context": "regionserver", "tag.Hostname": "a67e7d17e593", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732412774891, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T01:46:32,696 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37581 {}] master.MasterRpcServices(700): a67e7d17e593,40811,1732412774716 reported a fatal error: ***** ABORTING region server a67e7d17e593,40811,1732412774716: testing ***** 2024-11-24T01:46:32,697 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a67e7d17e593,40811,1732412774716' ***** 2024-11-24T01:46:32,697 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T01:46:32,698 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T01:46:32,698 INFO [RS:0;a67e7d17e593:40811 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T01:46:32,698 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T01:46:32,698 INFO [RS:0;a67e7d17e593:40811 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T01:46:32,698 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33283 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:48606 deadline: 1732412852698, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=40811 startCode=1732412774716. As of locationSeqNum=12. 2024-11-24T01:46:32,699 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(956): aborting server a67e7d17e593,40811,1732412774716 2024-11-24T01:46:32,699 INFO [RS:0;a67e7d17e593:40811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T01:46:32,699 INFO [RS:0;a67e7d17e593:40811 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a67e7d17e593:40811. 2024-11-24T01:46:32,699 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:32,699 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=40811 startCode=1732412774716. As of locationSeqNum=12. 2024-11-24T01:46:32,699 DEBUG [RS:0;a67e7d17e593:40811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T01:46:32,699 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=40811 startCode=1732412774716. As of locationSeqNum=12. 2024-11-24T01:46:32,700 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:32,700 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a67e7d17e593 port=40811 startCode=1732412774716. As of locationSeqNum=12. 2024-11-24T01:46:32,700 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,700 DEBUG [RS:0;a67e7d17e593:40811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:32,700 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T01:46:32,700 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1325): Online Regions={5bd22b2d59c54a42edcaf121e896c37c=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.} 2024-11-24T01:46:32,701 DEBUG [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1351): Waiting on 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:32,701 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,702 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412792699Running coprocessor pre-close hooks at 1732412792699Disabling compacts and flushes for region at 1732412792699Disabling writes for close at 1732412792700 (+1 ms)Writing region close event to WAL at 1732412792701 (+1 ms)Running coprocessor post-close hooks at 1732412792701Closed at 1732412792701 2024-11-24T01:46:32,702 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:32,762 INFO [regionserver/a67e7d17e593:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:32,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a67e7d17e593,40811,1732412774716 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:32,807 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a67e7d17e593,40811,1732412774716 aborting 2024-11-24T01:46:32,808 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a67e7d17e593,40811,1732412774716 aborting 2024-11-24T01:46:32,808 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=12 from cache 2024-11-24T01:46:32,901 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(976): stopping server a67e7d17e593,40811,1732412774716; all regions closed. 2024-11-24T01:46:32,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741834_1010 (size=2288) 2024-11-24T01:46:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741834_1010 (size=2288) 2024-11-24T01:46:32,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741834_1010 (size=2288) 2024-11-24T01:46:32,907 DEBUG [RS:0;a67e7d17e593:40811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:32,907 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:32,907 INFO [RS:0;a67e7d17e593:40811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T01:46:32,907 INFO [RS:0;a67e7d17e593:40811 {}] hbase.ChoreService(370): Chore service for: regionserver/a67e7d17e593:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T01:46:32,908 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T01:46:32,908 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T01:46:32,908 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T01:46:32,908 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T01:46:32,908 INFO [RS:0;a67e7d17e593:40811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T01:46:32,909 INFO [RS:0;a67e7d17e593:40811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40811 2024-11-24T01:46:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a67e7d17e593,40811,1732412774716 2024-11-24T01:46:32,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T01:46:32,970 INFO [RS:0;a67e7d17e593:40811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T01:46:32,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a67e7d17e593,40811,1732412774716] 2024-11-24T01:46:32,987 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a67e7d17e593,40811,1732412774716 already deleted, retry=false 2024-11-24T01:46:32,987 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a67e7d17e593,40811,1732412774716 on a67e7d17e593,37581,1732412773942 2024-11-24T01:46:32,991 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a67e7d17e593,40811,1732412774716, splitWal=true, meta=false 2024-11-24T01:46:32,993 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for a67e7d17e593,40811,1732412774716 (carryingMeta=false) a67e7d17e593,40811,1732412774716/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4b38e750[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T01:46:32,994 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a67e7d17e593,40811,1732412774716, splitWal=true, meta=false 2024-11-24T01:46:32,995 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): a67e7d17e593,40811,1732412774716 had 1 regions 2024-11-24T01:46:32,997 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a67e7d17e593,40811,1732412774716, splitWal=true, meta=false, isMeta: false 2024-11-24T01:46:32,998 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting 2024-11-24T01:46:32,999 INFO [PEWorker-2 {}] master.SplitWALManager(105): a67e7d17e593,40811,1732412774716 WAL count=1, meta=false 2024-11-24T01:46:33,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a67e7d17e593%2C40811%2C1732412774716.1732412776506}] 2024-11-24T01:46:33,007 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:33,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a67e7d17e593%2C40811%2C1732412774716.1732412776506, worker=a67e7d17e593,33283,1732412774831}] 2024-11-24T01:46:33,018 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18] 2024-11-24T01:46:33,019 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a67e7d17e593:40811 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a67e7d17e593/172.17.0.2:40811 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:33,020 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18, error=java.net.ConnectException: Call to address=a67e7d17e593:40811 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a67e7d17e593/172.17.0.2:40811 2024-11-24T01:46:33,020 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T01:46:33,020 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 from cache 2024-11-24T01:46:33,021 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address a67e7d17e593:40811 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a67e7d17e593/172.17.0.2:40811 2024-11-24T01:46:33,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:33,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40811-0x1016bcea5fe0001, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:33,080 INFO [RS:0;a67e7d17e593:40811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T01:46:33,080 INFO [RS:0;a67e7d17e593:40811 {}] regionserver.HRegionServer(1031): Exiting; stopping=a67e7d17e593,40811,1732412774716; zookeeper connection closed. 2024-11-24T01:46:33,080 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@500a4734 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@500a4734 2024-11-24T01:46:33,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33283 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T01:46:33,190 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506, size=2.2 K (2288bytes) 2024-11-24T01:46:33,191 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 2024-11-24T01:46:33,191 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 after 0ms 2024-11-24T01:46:33,219 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:33,220 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 took 30ms 2024-11-24T01:46:33,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 5bd22b2d59c54a42edcaf121e896c37c: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T01:46:33,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 so closing down 2024-11-24T01:46:33,227 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:33,227 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:33,228 INFO [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-24T01:46:33,228 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506, journal: Splitting hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506, size=2.2 K (2288bytes) at 1732412793191Finishing writing output for hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 so closing down at 1732412793227 (+36 ms)3 split writer threads finished at 1732412793228 (+1 ms)Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732412793228 2024-11-24T01:46:33,228 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 2024-11-24T01:46:33,229 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a67e7d17e593:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T01:46:33,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37581 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T01:46:33,237 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting/a67e7d17e593%2C40811%2C1732412774716.1732412776506 to hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs 2024-11-24T01:46:33,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T01:46:33,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a67e7d17e593%2C40811%2C1732412774716.1732412776506, worker=a67e7d17e593,33283,1732412774831 in 228 msec 2024-11-24T01:46:33,242 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:33,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T01:46:33,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a67e7d17e593%2C40811%2C1732412774716.1732412776506, worker=a67e7d17e593,33283,1732412774831 in 242 msec 2024-11-24T01:46:33,247 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting dir is empty, no logs to split. 2024-11-24T01:46:33,247 INFO [PEWorker-2 {}] master.SplitWALManager(105): a67e7d17e593,40811,1732412774716 WAL count=0, meta=false 2024-11-24T01:46:33,247 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if a67e7d17e593,40811,1732412774716 WAL splitting is done? wals=0, meta=false 2024-11-24T01:46:33,249 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for a67e7d17e593,40811,1732412774716 failed, ignore...File hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/WALs/a67e7d17e593,40811,1732412774716-splitting does not exist. 2024-11-24T01:46:33,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN}] 2024-11-24T01:46:33,254 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN 2024-11-24T01:46:33,255 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T01:46:33,328 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18] 2024-11-24T01:46:33,328 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to a67e7d17e593:40811 this server is in the failed servers list 2024-11-24T01:46:33,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a67e7d17e593:40811 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a67e7d17e593:40811 2024-11-24T01:46:33,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a67e7d17e593:40811 2024-11-24T01:46:33,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,40811,1732412774716, seqNum=18 from cache 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(204): Hosts are {a67e7d17e593=0} racks are {/default-rack=0} 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T01:46:33,406 INFO [a67e7d17e593:37581 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T01:46:33,406 INFO [a67e7d17e593:37581 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T01:46:33,406 DEBUG [a67e7d17e593:37581 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T01:46:33,406 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPENING, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:33,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN because future has completed 2024-11-24T01:46:33,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831}] 2024-11-24T01:46:33,571 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:33,571 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bd22b2d59c54a42edcaf121e896c37c, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:33,572 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,572 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:33,572 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,572 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,575 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,576 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf1 2024-11-24T01:46:33,576 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:33,582 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf1/d18a2e08aeef47bb85416f7f779a7159 2024-11-24T01:46:33,583 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:33,583 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,584 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bd22b2d59c54a42edcaf121e896c37c columnFamilyName cf2 2024-11-24T01:46:33,584 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:33,590 DEBUG [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/cf2/6a09724ed7b54a8da3aafdd364b12237 2024-11-24T01:46:33,590 INFO [StoreOpener-5bd22b2d59c54a42edcaf121e896c37c-1 {}] regionserver.HStore(327): Store=5bd22b2d59c54a42edcaf121e896c37c/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:33,590 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,591 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,592 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,593 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,593 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,593 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T01:46:33,595 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,595 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 5bd22b2d59c54a42edcaf121e896c37c; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62784084, jitterRate=-0.06444424390792847}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T01:46:33,595 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:33,596 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 5bd22b2d59c54a42edcaf121e896c37c: Running coprocessor pre-open hook at 1732412793572Writing region info on filesystem at 1732412793573 (+1 ms)Initializing all the Stores at 1732412793574 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412793574Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412793575 (+1 ms)Cleaning up temporary data from old regions at 1732412793593 (+18 ms)Running coprocessor post-open hooks at 1732412793595 (+2 ms)Region opened successfully at 1732412793596 (+1 ms) 2024-11-24T01:46:33,597 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., pid=17, masterSystemTime=1732412793563 2024-11-24T01:46:33,599 DEBUG [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:33,599 INFO [RS_OPEN_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:33,599 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=5bd22b2d59c54a42edcaf121e896c37c, regionState=OPEN, openSeqNum=18, regionLocation=a67e7d17e593,33283,1732412774831 2024-11-24T01:46:33,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 because future has completed 2024-11-24T01:46:33,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T01:46:33,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bd22b2d59c54a42edcaf121e896c37c, server=a67e7d17e593,33283,1732412774831 in 193 msec 2024-11-24T01:46:33,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T01:46:33,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=5bd22b2d59c54a42edcaf121e896c37c, ASSIGN in 354 msec 2024-11-24T01:46:33,608 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server a67e7d17e593,40811,1732412774716 after splitting done 2024-11-24T01:46:33,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure a67e7d17e593,40811,1732412774716, splitWal=true, meta=false in 620 msec 2024-11-24T01:46:33,842 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c., hostname=a67e7d17e593,33283,1732412774831, seqNum=18] 2024-11-24T01:46:33,856 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411 (was 416), OpenFileDescriptor=1047 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=7412 (was 7464) 2024-11-24T01:46:33,856 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1047 is superior to 1024 2024-11-24T01:46:33,866 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411, OpenFileDescriptor=1047, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=7411 2024-11-24T01:46:33,866 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1047 is superior to 1024 2024-11-24T01:46:33,879 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:33,881 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:33,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:33,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76115035, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-76115035, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:33,896 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76115035/hregion-76115035.1732412793885, exclude list is [], retry=0 2024-11-24T01:46:33,899 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:33,899 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:33,899 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:33,925 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76115035/hregion-76115035.1732412793885 2024-11-24T01:46:33,925 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:33,925 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 506c8802fb9b4ebc8f7417527187626a, NAME => 'testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741893_1071 (size=67) 2024-11-24T01:46:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741893_1071 (size=67) 2024-11-24T01:46:33,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741893_1071 (size=67) 2024-11-24T01:46:33,936 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:33,938 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,940 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName a 2024-11-24T01:46:33,940 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:33,941 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:33,941 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,943 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName b 2024-11-24T01:46:33,943 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:33,943 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:33,943 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,945 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName c 2024-11-24T01:46:33,945 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:33,945 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:33,946 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,946 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,947 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,948 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,948 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,948 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:33,949 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:33,951 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:33,952 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 506c8802fb9b4ebc8f7417527187626a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71719286, jitterRate=0.0687006413936615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:33,953 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 506c8802fb9b4ebc8f7417527187626a: Writing region info on filesystem at 1732412793936Initializing all the Stores at 1732412793937 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412793937Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412793938 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412793938Cleaning up temporary data from old regions at 1732412793948 (+10 ms)Region opened successfully at 1732412793953 (+5 ms) 2024-11-24T01:46:33,953 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 506c8802fb9b4ebc8f7417527187626a, disabling compactions & flushes 2024-11-24T01:46:33,953 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:33,953 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:33,953 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. after waiting 0 ms 2024-11-24T01:46:33,953 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:33,954 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:33,954 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 506c8802fb9b4ebc8f7417527187626a: Waiting for close lock at 1732412793953Disabling compacts and flushes for region at 1732412793953Disabling writes for close at 1732412793953Writing region close event to WAL at 1732412793953Closed at 1732412793954 (+1 ms) 2024-11-24T01:46:33,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741892_1070 (size=175) 2024-11-24T01:46:33,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741892_1070 (size=175) 2024-11-24T01:46:33,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741892_1070 (size=175) 2024-11-24T01:46:33,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:33,961 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76115035:(num 1732412793885) 2024-11-24T01:46:33,961 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:33,962 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:33,974 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, exclude list is [], retry=0 2024-11-24T01:46:33,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:33,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:33,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:34,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 2024-11-24T01:46:34,003 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 506c8802fb9b4ebc8f7417527187626a, NAME => 'testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,003 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,004 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,005 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName a 2024-11-24T01:46:34,006 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,006 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,006 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,007 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName b 2024-11-24T01:46:34,007 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,008 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,008 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,009 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName c 2024-11-24T01:46:34,009 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,009 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,009 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,010 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,011 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,012 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,012 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,013 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:34,014 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,016 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 506c8802fb9b4ebc8f7417527187626a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63342142, jitterRate=-0.05612853169441223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:34,017 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 506c8802fb9b4ebc8f7417527187626a: Writing region info on filesystem at 1732412794003Initializing all the Stores at 1732412794004 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794004Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794004Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794004Cleaning up temporary data from old regions at 1732412794012 (+8 ms)Region opened successfully at 1732412794017 (+5 ms) 2024-11-24T01:46:34,041 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 506c8802fb9b4ebc8f7417527187626a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T01:46:34,056 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/a/7804aa4fb37c4614892af8707b275ee5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732412794017/Put/seqid=0 2024-11-24T01:46:34,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741895_1073 (size=5958) 2024-11-24T01:46:34,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741895_1073 (size=5958) 2024-11-24T01:46:34,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741895_1073 (size=5958) 2024-11-24T01:46:34,063 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/a/7804aa4fb37c4614892af8707b275ee5 2024-11-24T01:46:34,088 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/ad6d631c673b466c95990ba9553340fa is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732412794025/Put/seqid=0 2024-11-24T01:46:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741896_1074 (size=5958) 2024-11-24T01:46:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741896_1074 (size=5958) 2024-11-24T01:46:34,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741896_1074 (size=5958) 2024-11-24T01:46:34,095 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/ad6d631c673b466c95990ba9553340fa 2024-11-24T01:46:34,115 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/c/c2820865859e4ae3a733a9731e798ae5 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732412794032/Put/seqid=0 2024-11-24T01:46:34,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741897_1075 (size=5958) 2024-11-24T01:46:34,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741897_1075 (size=5958) 2024-11-24T01:46:34,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741897_1075 (size=5958) 2024-11-24T01:46:34,121 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/c/c2820865859e4ae3a733a9731e798ae5 2024-11-24T01:46:34,127 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/a/7804aa4fb37c4614892af8707b275ee5 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/a/7804aa4fb37c4614892af8707b275ee5 2024-11-24T01:46:34,131 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/a/7804aa4fb37c4614892af8707b275ee5, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T01:46:34,133 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/ad6d631c673b466c95990ba9553340fa as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/ad6d631c673b466c95990ba9553340fa 2024-11-24T01:46:34,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/ad6d631c673b466c95990ba9553340fa, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T01:46:34,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/c/c2820865859e4ae3a733a9731e798ae5 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/c/c2820865859e4ae3a733a9731e798ae5 2024-11-24T01:46:34,144 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/c/c2820865859e4ae3a733a9731e798ae5, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T01:46:34,145 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 506c8802fb9b4ebc8f7417527187626a in 104ms, sequenceid=33, compaction requested=false 2024-11-24T01:46:34,145 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 506c8802fb9b4ebc8f7417527187626a: 2024-11-24T01:46:34,146 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 506c8802fb9b4ebc8f7417527187626a, disabling compactions & flushes 2024-11-24T01:46:34,146 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:34,146 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:34,146 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. after waiting 0 ms 2024-11-24T01:46:34,146 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:34,147 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. 2024-11-24T01:46:34,147 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 506c8802fb9b4ebc8f7417527187626a: Waiting for close lock at 1732412794146Disabling compacts and flushes for region at 1732412794146Disabling writes for close at 1732412794146Writing region close event to WAL at 1732412794147 (+1 ms)Closed at 1732412794147 2024-11-24T01:46:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741894_1072 (size=6311) 2024-11-24T01:46:34,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741894_1072 (size=6311) 2024-11-24T01:46:34,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741894_1072 (size=6311) 2024-11-24T01:46:34,154 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/ad6d631c673b466c95990ba9553340fa to hdfs://localhost:34675/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/ad6d631c673b466c95990ba9553340fa 2024-11-24T01:46:34,169 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, size=6.2 K (6311bytes) 2024-11-24T01:46:34,169 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 2024-11-24T01:46:34,170 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 after 0ms 2024-11-24T01:46:34,197 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:34,197 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 took 28ms 2024-11-24T01:46:34,199 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 so closing down 2024-11-24T01:46:34,199 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:34,200 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732412793963.temp 2024-11-24T01:46:34,223 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp 2024-11-24T01:46:34,224 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:34,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741898_1076 (size=5575) 2024-11-24T01:46:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741898_1076 (size=5575) 2024-11-24T01:46:34,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741898_1076 (size=5575) 2024-11-24T01:46:34,230 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:34,231 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 2024-11-24T01:46:34,232 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-24T01:46:34,232 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, journal: Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, size=6.2 K (6311bytes) at 1732412794169Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 so closing down at 1732412794199 (+30 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp at 1732412794224 (+25 ms)3 split writer threads finished at 1732412794224Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732412794230 (+6 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000003-wal.1732412793963.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 at 1732412794231 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732412794232 (+1 ms) 2024-11-24T01:46:34,233 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412793963 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412793963 2024-11-24T01:46:34,234 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 2024-11-24T01:46:34,234 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:34,235 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:34,248 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412794235, exclude list is [], retry=0 2024-11-24T01:46:34,251 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:34,251 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:34,251 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:34,277 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412794235 2024-11-24T01:46:34,277 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:34,277 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 506c8802fb9b4ebc8f7417527187626a, NAME => 'testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:34,278 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:34,278 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,278 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,279 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,280 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName a 2024-11-24T01:46:34,280 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,285 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/a/7804aa4fb37c4614892af8707b275ee5 2024-11-24T01:46:34,285 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,285 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,287 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName b 2024-11-24T01:46:34,287 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,287 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,287 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,288 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 506c8802fb9b4ebc8f7417527187626a columnFamilyName c 2024-11-24T01:46:34,288 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T01:46:34,291 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:34,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T01:46:34,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:34,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T01:46:34,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:34,295 DEBUG [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/c/c2820865859e4ae3a733a9731e798ae5 2024-11-24T01:46:34,295 INFO [StoreOpener-506c8802fb9b4ebc8f7417527187626a-1 {}] regionserver.HStore(327): Store=506c8802fb9b4ebc8f7417527187626a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,295 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,296 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,298 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 2024-11-24T01:46:34,325 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:34,327 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 2024-11-24T01:46:34,327 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 506c8802fb9b4ebc8f7417527187626a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T01:46:34,341 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/fd50763ab5694132a03d7b55ea44bc89 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732412794025/Put/seqid=0 2024-11-24T01:46:34,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741900_1078 (size=5958) 2024-11-24T01:46:34,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741900_1078 (size=5958) 2024-11-24T01:46:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741900_1078 (size=5958) 2024-11-24T01:46:34,348 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/fd50763ab5694132a03d7b55ea44bc89 2024-11-24T01:46:34,354 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/.tmp/b/fd50763ab5694132a03d7b55ea44bc89 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/fd50763ab5694132a03d7b55ea44bc89 2024-11-24T01:46:34,360 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/b/fd50763ab5694132a03d7b55ea44bc89, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T01:46:34,361 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 506c8802fb9b4ebc8f7417527187626a in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T01:46:34,361 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/0000000000000000032 2024-11-24T01:46:34,363 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,363 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,363 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:34,365 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 506c8802fb9b4ebc8f7417527187626a 2024-11-24T01:46:34,367 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/506c8802fb9b4ebc8f7417527187626a/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T01:46:34,368 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 506c8802fb9b4ebc8f7417527187626a; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68039295, jitterRate=0.013864502310752869}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:34,369 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 506c8802fb9b4ebc8f7417527187626a: Writing region info on filesystem at 1732412794278Initializing all the Stores at 1732412794279 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794279Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794279Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794279Obtaining lock to block concurrent updates at 1732412794327 (+48 ms)Preparing flush snapshotting stores in 506c8802fb9b4ebc8f7417527187626a at 1732412794327Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732412794327Flushing stores of testReplayEditsWrittenViaHRegion,,1732412793880.506c8802fb9b4ebc8f7417527187626a. at 1732412794327Flushing 506c8802fb9b4ebc8f7417527187626a/b: creating writer at 1732412794327Flushing 506c8802fb9b4ebc8f7417527187626a/b: appending metadata at 1732412794341 (+14 ms)Flushing 506c8802fb9b4ebc8f7417527187626a/b: closing flushed file at 1732412794341Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51a7217: reopening flushed file at 1732412794353 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 506c8802fb9b4ebc8f7417527187626a in 33ms, sequenceid=32, compaction requested=false; wal=null at 1732412794361 (+8 ms)Cleaning up temporary data from old regions at 1732412794363 (+2 ms)Region opened successfully at 1732412794369 (+6 ms) 2024-11-24T01:46:34,392 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 411) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:46668 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39128 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:39244 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:41886 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:48584 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:53130 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1121 (was 1047) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=7407 (was 7411) 2024-11-24T01:46:34,392 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-11-24T01:46:34,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1121, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=7404 2024-11-24T01:46:34,405 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1121 is superior to 1024 2024-11-24T01:46:34,419 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:34,422 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:34,422 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:34,425 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57836826, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-57836826, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:34,437 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57836826/hregion-57836826.1732412794425, exclude list is [], retry=0 2024-11-24T01:46:34,440 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:34,440 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:34,441 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:34,466 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57836826/hregion-57836826.1732412794425 2024-11-24T01:46:34,466 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:34,466 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bd9cd728d00ba32abdbe973972571832, NAME => 'testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:34,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741902_1080 (size=68) 2024-11-24T01:46:34,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741902_1080 (size=68) 2024-11-24T01:46:34,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741902_1080 (size=68) 2024-11-24T01:46:34,475 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:34,477 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,478 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName a 2024-11-24T01:46:34,479 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,479 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,479 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,481 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName b 2024-11-24T01:46:34,481 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,481 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,481 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,483 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName c 2024-11-24T01:46:34,483 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,483 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,483 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,484 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,485 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,486 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,486 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,486 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:34,487 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,489 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:34,490 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd9cd728d00ba32abdbe973972571832; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59701224, jitterRate=-0.11038243770599365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:34,491 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd9cd728d00ba32abdbe973972571832: Writing region info on filesystem at 1732412794476Initializing all the Stores at 1732412794476Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794476Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794477 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794477Cleaning up temporary data from old regions at 1732412794486 (+9 ms)Region opened successfully at 1732412794491 (+5 ms) 2024-11-24T01:46:34,491 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bd9cd728d00ba32abdbe973972571832, disabling compactions & flushes 2024-11-24T01:46:34,491 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:34,491 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:34,491 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. after waiting 0 ms 2024-11-24T01:46:34,491 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:34,492 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:34,492 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bd9cd728d00ba32abdbe973972571832: Waiting for close lock at 1732412794491Disabling compacts and flushes for region at 1732412794491Disabling writes for close at 1732412794491Writing region close event to WAL at 1732412794492 (+1 ms)Closed at 1732412794492 2024-11-24T01:46:34,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741901_1079 (size=175) 2024-11-24T01:46:34,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741901_1079 (size=175) 2024-11-24T01:46:34,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741901_1079 (size=175) 2024-11-24T01:46:34,496 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:34,496 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57836826:(num 1732412794425) 2024-11-24T01:46:34,496 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:34,498 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:34,509 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, exclude list is [], retry=0 2024-11-24T01:46:34,511 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:34,512 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:34,512 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:34,536 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 2024-11-24T01:46:34,536 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:34,591 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bd9cd728d00ba32abdbe973972571832, NAME => 'testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:34,594 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,594 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:34,594 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,594 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,596 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,597 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName a 2024-11-24T01:46:34,598 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,598 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,598 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,599 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName b 2024-11-24T01:46:34,599 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,600 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,600 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,601 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName c 2024-11-24T01:46:34,601 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:34,601 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:34,601 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,602 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,603 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,605 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,605 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,605 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:34,606 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,607 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd9cd728d00ba32abdbe973972571832; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64348174, jitterRate=-0.041137486696243286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:34,607 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:34,608 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd9cd728d00ba32abdbe973972571832: Running coprocessor pre-open hook at 1732412794594Writing region info on filesystem at 1732412794594Initializing all the Stores at 1732412794596 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794596Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794596Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412794596Cleaning up temporary data from old regions at 1732412794605 (+9 ms)Running coprocessor post-open hooks at 1732412794607 (+2 ms)Region opened successfully at 1732412794608 (+1 ms) 2024-11-24T01:46:34,619 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd9cd728d00ba32abdbe973972571832 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T01:46:34,620 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:35,621 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:36,291 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T01:46:36,622 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:37,625 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:38,625 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:39,627 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:40,629 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:41,631 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:42,632 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:43,116 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T01:46:43,633 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for bd9cd728d00ba32abdbe973972571832/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:43,634 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd9cd728d00ba32abdbe973972571832: 2024-11-24T01:46:43,634 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd9cd728d00ba32abdbe973972571832: 2024-11-24T01:46:43,646 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bd9cd728d00ba32abdbe973972571832, disabling compactions & flushes 2024-11-24T01:46:43,646 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. after waiting 0 ms 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. 2024-11-24T01:46:43,646 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bd9cd728d00ba32abdbe973972571832: Waiting for close lock at 1732412803646Running coprocessor pre-close hooks at 1732412803646Disabling compacts and flushes for region at 1732412803646Disabling writes for close at 1732412803646Writing region close event to WAL at 1732412803646Running coprocessor post-close hooks at 1732412803646Closed at 1732412803646 2024-11-24T01:46:43,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741903_1081 (size=4236) 2024-11-24T01:46:43,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741903_1081 (size=4236) 2024-11-24T01:46:43,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741903_1081 (size=4236) 2024-11-24T01:46:43,663 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, size=4.1 K (4236bytes) 2024-11-24T01:46:43,663 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 2024-11-24T01:46:43,663 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 after 0ms 2024-11-24T01:46:43,687 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:43,687 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 took 25ms 2024-11-24T01:46:43,690 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 so closing down 2024-11-24T01:46:43,690 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:43,691 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732412794498.temp 2024-11-24T01:46:43,714 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp 2024-11-24T01:46:43,714 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:43,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741904_1082 (size=3245) 2024-11-24T01:46:43,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741904_1082 (size=3245) 2024-11-24T01:46:43,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741904_1082 (size=3245) 2024-11-24T01:46:43,721 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:43,722 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 2024-11-24T01:46:43,722 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-24T01:46:43,722 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, journal: Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, size=4.1 K (4236bytes) at 1732412803663Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 so closing down at 1732412803690 (+27 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp at 1732412803714 (+24 ms)3 split writer threads finished at 1732412803714Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732412803721 (+7 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000004-wal.1732412794498.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 at 1732412803722 (+1 ms)Processed 23 edits across 1 Regions in 34 ms; skipped=3; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732412803722 2024-11-24T01:46:43,724 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412794498 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412794498 2024-11-24T01:46:43,725 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 2024-11-24T01:46:43,725 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:43,727 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:43,738 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412803728, exclude list is [], retry=0 2024-11-24T01:46:43,741 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:43,741 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:43,741 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:43,764 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412803728 2024-11-24T01:46:43,765 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:43,765 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bd9cd728d00ba32abdbe973972571832, NAME => 'testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:43,765 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,765 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:43,766 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,766 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,767 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,768 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName a 2024-11-24T01:46:43,768 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:43,769 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:43,769 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,769 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName b 2024-11-24T01:46:43,769 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:43,770 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:43,770 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,770 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd9cd728d00ba32abdbe973972571832 columnFamilyName c 2024-11-24T01:46:43,770 DEBUG [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:43,771 INFO [StoreOpener-bd9cd728d00ba32abdbe973972571832-1 {}] regionserver.HStore(327): Store=bd9cd728d00ba32abdbe973972571832/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:43,771 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,772 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,773 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,773 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 2024-11-24T01:46:43,800 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:43,801 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 2024-11-24T01:46:43,802 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd9cd728d00ba32abdbe973972571832 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T01:46:43,816 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/a/033e69c4e28144559417a3999d2a30ca is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732412803639/Put/seqid=0 2024-11-24T01:46:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741906_1084 (size=5523) 2024-11-24T01:46:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741906_1084 (size=5523) 2024-11-24T01:46:43,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741906_1084 (size=5523) 2024-11-24T01:46:43,823 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/a/033e69c4e28144559417a3999d2a30ca 2024-11-24T01:46:43,842 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/b/860931fffd7649dbbea42003967ba3f4 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732412803635/Put/seqid=0 2024-11-24T01:46:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741907_1085 (size=5524) 2024-11-24T01:46:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741907_1085 (size=5524) 2024-11-24T01:46:43,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741907_1085 (size=5524) 2024-11-24T01:46:43,850 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/b/860931fffd7649dbbea42003967ba3f4 2024-11-24T01:46:43,869 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/c/e7150f2509714c568c43c177128dd490 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732412803637/Put/seqid=0 2024-11-24T01:46:43,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741908_1086 (size=5457) 2024-11-24T01:46:43,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741908_1086 (size=5457) 2024-11-24T01:46:43,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741908_1086 (size=5457) 2024-11-24T01:46:43,876 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/c/e7150f2509714c568c43c177128dd490 2024-11-24T01:46:43,882 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/a/033e69c4e28144559417a3999d2a30ca as hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/a/033e69c4e28144559417a3999d2a30ca 2024-11-24T01:46:43,888 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/a/033e69c4e28144559417a3999d2a30ca, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T01:46:43,889 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/b/860931fffd7649dbbea42003967ba3f4 as hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/b/860931fffd7649dbbea42003967ba3f4 2024-11-24T01:46:43,895 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/b/860931fffd7649dbbea42003967ba3f4, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T01:46:43,897 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/.tmp/c/e7150f2509714c568c43c177128dd490 as hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/c/e7150f2509714c568c43c177128dd490 2024-11-24T01:46:43,903 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/c/e7150f2509714c568c43c177128dd490, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T01:46:43,903 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bd9cd728d00ba32abdbe973972571832 in 102ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T01:46:43,904 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/0000000000000000026 2024-11-24T01:46:43,905 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,905 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,906 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:43,907 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,909 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsAfterAbortingFlush/bd9cd728d00ba32abdbe973972571832/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T01:46:43,910 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bd9cd728d00ba32abdbe973972571832; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72625719, jitterRate=0.08220754563808441}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:43,910 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd9cd728d00ba32abdbe973972571832 2024-11-24T01:46:43,911 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bd9cd728d00ba32abdbe973972571832: Running coprocessor pre-open hook at 1732412803766Writing region info on filesystem at 1732412803766Initializing all the Stores at 1732412803767 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412803767Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412803767Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412803767Obtaining lock to block concurrent updates at 1732412803802 (+35 ms)Preparing flush snapshotting stores in bd9cd728d00ba32abdbe973972571832 at 1732412803802Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732412803802Flushing stores of testReplayEditsAfterAbortingFlush,,1732412794420.bd9cd728d00ba32abdbe973972571832. at 1732412803802Flushing bd9cd728d00ba32abdbe973972571832/a: creating writer at 1732412803802Flushing bd9cd728d00ba32abdbe973972571832/a: appending metadata at 1732412803815 (+13 ms)Flushing bd9cd728d00ba32abdbe973972571832/a: closing flushed file at 1732412803815Flushing bd9cd728d00ba32abdbe973972571832/b: creating writer at 1732412803828 (+13 ms)Flushing bd9cd728d00ba32abdbe973972571832/b: appending metadata at 1732412803841 (+13 ms)Flushing bd9cd728d00ba32abdbe973972571832/b: closing flushed file at 1732412803841Flushing bd9cd728d00ba32abdbe973972571832/c: creating writer at 1732412803855 (+14 ms)Flushing bd9cd728d00ba32abdbe973972571832/c: appending metadata at 1732412803868 (+13 ms)Flushing bd9cd728d00ba32abdbe973972571832/c: closing flushed file at 1732412803868Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31096a2: reopening flushed file at 1732412803881 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358880: reopening flushed file at 1732412803888 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54a4a9c2: reopening flushed file at 1732412803896 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for bd9cd728d00ba32abdbe973972571832 in 102ms, sequenceid=26, compaction requested=false; wal=null at 1732412803903 (+7 ms)Cleaning up temporary data from old regions at 1732412803905 (+2 ms)Running coprocessor post-open hooks at 1732412803910 (+5 ms)Region opened successfully at 1732412803911 (+1 ms) 2024-11-24T01:46:43,932 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 418) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:51836 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34624 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60050 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60044 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:51844 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34640 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1178 (was 1121) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=224 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=7396 (was 7404) 2024-11-24T01:46:43,932 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-24T01:46:43,945 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1178, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=7395 2024-11-24T01:46:43,945 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1178 is superior to 1024 2024-11-24T01:46:43,965 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:43,967 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:43,968 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:43,970 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-28544652, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-28544652, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:43,987 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-28544652/hregion-28544652.1732412803970, exclude list is [], retry=0 2024-11-24T01:46:43,989 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:43,989 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:43,990 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:44,014 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-28544652/hregion-28544652.1732412803970 2024-11-24T01:46:44,014 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:44,014 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 258fec01e0f59059eed2650e1f4267b7, NAME => 'testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741910_1088 (size=61) 2024-11-24T01:46:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741910_1088 (size=61) 2024-11-24T01:46:44,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741910_1088 (size=61) 2024-11-24T01:46:44,023 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,024 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,026 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 258fec01e0f59059eed2650e1f4267b7 columnFamilyName a 2024-11-24T01:46:44,026 DEBUG [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,026 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(327): Store=258fec01e0f59059eed2650e1f4267b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,026 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,028 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,028 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,030 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,032 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:44,032 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 258fec01e0f59059eed2650e1f4267b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64495086, jitterRate=-0.03894832730293274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 258fec01e0f59059eed2650e1f4267b7: Writing region info on filesystem at 1732412804023Initializing all the Stores at 1732412804024 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804024Cleaning up temporary data from old regions at 1732412804028 (+4 ms)Region opened successfully at 1732412804032 (+4 ms) 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 258fec01e0f59059eed2650e1f4267b7, disabling compactions & flushes 2024-11-24T01:46:44,033 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. after waiting 0 ms 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,033 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,033 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 258fec01e0f59059eed2650e1f4267b7: Waiting for close lock at 1732412804033Disabling compacts and flushes for region at 1732412804033Disabling writes for close at 1732412804033Writing region close event to WAL at 1732412804033Closed at 1732412804033 2024-11-24T01:46:44,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741909_1087 (size=175) 2024-11-24T01:46:44,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741909_1087 (size=175) 2024-11-24T01:46:44,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741909_1087 (size=175) 2024-11-24T01:46:44,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:44,038 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-28544652:(num 1732412803970) 2024-11-24T01:46:44,038 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:44,040 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:44,054 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, exclude list is [], retry=0 2024-11-24T01:46:44,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:44,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:44,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:44,082 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 2024-11-24T01:46:44,082 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:44,082 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 258fec01e0f59059eed2650e1f4267b7, NAME => 'testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:44,082 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,083 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,083 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,084 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,085 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 258fec01e0f59059eed2650e1f4267b7 columnFamilyName a 2024-11-24T01:46:44,085 DEBUG [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,085 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(327): Store=258fec01e0f59059eed2650e1f4267b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,085 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,087 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,088 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,088 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,090 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,091 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 258fec01e0f59059eed2650e1f4267b7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60570051, jitterRate=-0.09743590652942657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:44,092 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 258fec01e0f59059eed2650e1f4267b7: Writing region info on filesystem at 1732412804083Initializing all the Stores at 1732412804084 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804084Cleaning up temporary data from old regions at 1732412804088 (+4 ms)Region opened successfully at 1732412804091 (+3 ms) 2024-11-24T01:46:44,101 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 258fec01e0f59059eed2650e1f4267b7, disabling compactions & flushes 2024-11-24T01:46:44,101 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,101 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,102 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. after waiting 0 ms 2024-11-24T01:46:44,102 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,102 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,102 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. 2024-11-24T01:46:44,102 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 258fec01e0f59059eed2650e1f4267b7: Waiting for close lock at 1732412804101Disabling compacts and flushes for region at 1732412804101Disabling writes for close at 1732412804102 (+1 ms)Writing region close event to WAL at 1732412804102Closed at 1732412804102 2024-11-24T01:46:44,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741911_1089 (size=1795) 2024-11-24T01:46:44,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741911_1089 (size=1795) 2024-11-24T01:46:44,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741911_1089 (size=1795) 2024-11-24T01:46:44,119 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, size=1.8 K (1795bytes) 2024-11-24T01:46:44,119 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 2024-11-24T01:46:44,119 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 after 0ms 2024-11-24T01:46:44,143 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:44,143 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 took 24ms 2024-11-24T01:46:44,164 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 so closing down 2024-11-24T01:46:44,164 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:44,165 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732412804040.temp 2024-11-24T01:46:44,189 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp 2024-11-24T01:46:44,189 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741912_1090 (size=1795) 2024-11-24T01:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741912_1090 (size=1795) 2024-11-24T01:46:44,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741912_1090 (size=1795) 2024-11-24T01:46:44,196 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:44,197 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp to hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 2024-11-24T01:46:44,197 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-24T01:46:44,197 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, journal: Splitting hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, size=1.8 K (1795bytes) at 1732412804119Finishing writing output for hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 so closing down at 1732412804164 (+45 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp at 1732412804189 (+25 ms)3 split writer threads finished at 1732412804189Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732412804196 (+7 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000003-wal.1732412804040.temp to hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 at 1732412804197 (+1 ms)Processed 10 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732412804197 2024-11-24T01:46:44,198 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804040 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412804040 2024-11-24T01:46:44,199 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 2024-11-24T01:46:44,226 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:44,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T01:46:44,291 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:44,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T01:46:44,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:44,543 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:44,545 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:44,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804546, exclude list is [], retry=0 2024-11-24T01:46:44,559 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:44,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:44,560 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:44,582 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804546 2024-11-24T01:46:44,583 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:44,583 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 258fec01e0f59059eed2650e1f4267b7, NAME => 'testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:44,583 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,583 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,583 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,586 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,587 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 258fec01e0f59059eed2650e1f4267b7 columnFamilyName a 2024-11-24T01:46:44,587 DEBUG [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,588 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(327): Store=258fec01e0f59059eed2650e1f4267b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,588 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,589 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,591 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 2024-11-24T01:46:44,618 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:44,619 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 2024-11-24T01:46:44,620 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 258fec01e0f59059eed2650e1f4267b7 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T01:46:44,638 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/.tmp/a/330715dd1d2c4511b6b5dfc8b46020e3 is 79, key is testDatalossWhenInputError/a:x0/1732412804092/Put/seqid=0 2024-11-24T01:46:44,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741914_1092 (size=5808) 2024-11-24T01:46:44,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741914_1092 (size=5808) 2024-11-24T01:46:44,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741914_1092 (size=5808) 2024-11-24T01:46:44,644 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/.tmp/a/330715dd1d2c4511b6b5dfc8b46020e3 2024-11-24T01:46:44,654 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/.tmp/a/330715dd1d2c4511b6b5dfc8b46020e3 as hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/a/330715dd1d2c4511b6b5dfc8b46020e3 2024-11-24T01:46:44,662 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/a/330715dd1d2c4511b6b5dfc8b46020e3, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T01:46:44,662 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 258fec01e0f59059eed2650e1f4267b7 in 43ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T01:46:44,663 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/0000000000000000012 2024-11-24T01:46:44,664 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,664 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,666 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,669 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T01:46:44,670 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 258fec01e0f59059eed2650e1f4267b7; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62589420, jitterRate=-0.06734496355056763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:44,670 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 258fec01e0f59059eed2650e1f4267b7: Writing region info on filesystem at 1732412804583Initializing all the Stores at 1732412804585 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804585Obtaining lock to block concurrent updates at 1732412804620 (+35 ms)Preparing flush snapshotting stores in 258fec01e0f59059eed2650e1f4267b7 at 1732412804620Finished memstore snapshotting testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732412804620Flushing stores of testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7. at 1732412804620Flushing 258fec01e0f59059eed2650e1f4267b7/a: creating writer at 1732412804620Flushing 258fec01e0f59059eed2650e1f4267b7/a: appending metadata at 1732412804637 (+17 ms)Flushing 258fec01e0f59059eed2650e1f4267b7/a: closing flushed file at 1732412804637Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b79498f: reopening flushed file at 1732412804652 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 258fec01e0f59059eed2650e1f4267b7 in 43ms, sequenceid=12, compaction requested=false; wal=null at 1732412804662 (+10 ms)Cleaning up temporary data from old regions at 1732412804664 (+2 ms)Region opened successfully at 1732412804670 (+6 ms) 2024-11-24T01:46:44,673 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 258fec01e0f59059eed2650e1f4267b7, NAME => 'testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:44,673 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732412803965.258fec01e0f59059eed2650e1f4267b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,673 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,674 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,675 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,676 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 258fec01e0f59059eed2650e1f4267b7 columnFamilyName a 2024-11-24T01:46:44,676 DEBUG [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,681 DEBUG [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/a/330715dd1d2c4511b6b5dfc8b46020e3 2024-11-24T01:46:44,681 INFO [StoreOpener-258fec01e0f59059eed2650e1f4267b7-1 {}] regionserver.HStore(327): Store=258fec01e0f59059eed2650e1f4267b7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,681 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,685 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 258fec01e0f59059eed2650e1f4267b7 2024-11-24T01:46:44,687 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testDatalossWhenInputError/258fec01e0f59059eed2650e1f4267b7/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T01:46:44,688 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 258fec01e0f59059eed2650e1f4267b7; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63699351, jitterRate=-0.050805702805519104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T01:46:44,688 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 258fec01e0f59059eed2650e1f4267b7: Writing region info on filesystem at 1732412804674Initializing all the Stores at 1732412804674Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804674Cleaning up temporary data from old regions at 1732412804683 (+9 ms)Region opened successfully at 1732412804688 (+5 ms) 2024-11-24T01:46:44,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34624 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60050 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:51844 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:51910 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34716 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60120 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1260 (was 1178) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=224 (was 224), ProcessCount=11 (was 11), AvailableMemoryMB=7372 (was 7395) 2024-11-24T01:46:44,704 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-24T01:46:44,716 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1260, MaxFileDescriptor=1048576, SystemLoadAverage=224, ProcessCount=11, AvailableMemoryMB=7372 2024-11-24T01:46:44,716 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1260 is superior to 1024 2024-11-24T01:46:44,730 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:44,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:44,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:44,734 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-83570779, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-83570779, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:44,746 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-83570779/hregion-83570779.1732412804735, exclude list is [], retry=0 2024-11-24T01:46:44,749 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:44,749 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:44,749 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:44,773 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-83570779/hregion-83570779.1732412804735 2024-11-24T01:46:44,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:44,773 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a386e431248dccab721bfd7e385c379c, NAME => 'testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741916_1094 (size=63) 2024-11-24T01:46:44,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741916_1094 (size=63) 2024-11-24T01:46:44,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741916_1094 (size=63) 2024-11-24T01:46:44,785 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,786 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,788 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName a 2024-11-24T01:46:44,788 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,788 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,789 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,790 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName b 2024-11-24T01:46:44,790 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,790 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,791 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,792 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName c 2024-11-24T01:46:44,792 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,792 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,792 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,793 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,795 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:44,796 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,798 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:44,798 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a386e431248dccab721bfd7e385c379c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75327142, jitterRate=0.12246188521385193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a386e431248dccab721bfd7e385c379c: Writing region info on filesystem at 1732412804785Initializing all the Stores at 1732412804786 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804786Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804786Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804786Cleaning up temporary data from old regions at 1732412804794 (+8 ms)Region opened successfully at 1732412804799 (+5 ms) 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a386e431248dccab721bfd7e385c379c, disabling compactions & flushes 2024-11-24T01:46:44,799 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. after waiting 0 ms 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:44,799 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:44,799 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a386e431248dccab721bfd7e385c379c: Waiting for close lock at 1732412804799Disabling compacts and flushes for region at 1732412804799Disabling writes for close at 1732412804799Writing region close event to WAL at 1732412804799Closed at 1732412804799 2024-11-24T01:46:44,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741915_1093 (size=175) 2024-11-24T01:46:44,802 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-83570779/hregion-83570779.1732412804735 not finished, retry = 0 2024-11-24T01:46:44,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741915_1093 (size=175) 2024-11-24T01:46:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741915_1093 (size=175) 2024-11-24T01:46:44,905 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:44,905 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-83570779:(num 1732412804735) 2024-11-24T01:46:44,906 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:44,908 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:44,926 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, exclude list is [], retry=0 2024-11-24T01:46:44,929 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:44,929 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:44,930 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:44,956 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 2024-11-24T01:46:44,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a386e431248dccab721bfd7e385c379c, NAME => 'testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,957 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,959 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,960 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName a 2024-11-24T01:46:44,960 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,960 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,960 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,961 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName b 2024-11-24T01:46:44,961 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,962 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,962 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,963 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName c 2024-11-24T01:46:44,963 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:44,963 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:44,963 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,964 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,967 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,967 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,967 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:44,969 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:44,970 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a386e431248dccab721bfd7e385c379c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69263822, jitterRate=0.03211137652397156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:44,971 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a386e431248dccab721bfd7e385c379c: Writing region info on filesystem at 1732412804957Initializing all the Stores at 1732412804958 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804958Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804959 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412804959Cleaning up temporary data from old regions at 1732412804967 (+8 ms)Region opened successfully at 1732412804971 (+4 ms) 2024-11-24T01:46:44,976 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732412804975/Put/seqid=0 2024-11-24T01:46:44,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741918_1096 (size=4875) 2024-11-24T01:46:44,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741918_1096 (size=4875) 2024-11-24T01:46:44,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741918_1096 (size=4875) 2024-11-24T01:46:45,389 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732412805388/Put/seqid=0 2024-11-24T01:46:45,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741919_1097 (size=4875) 2024-11-24T01:46:45,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741919_1097 (size=4875) 2024-11-24T01:46:45,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741919_1097 (size=4875) 2024-11-24T01:46:45,399 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732412805399/Put/seqid=0 2024-11-24T01:46:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741920_1098 (size=4875) 2024-11-24T01:46:45,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741920_1098 (size=4875) 2024-11-24T01:46:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741920_1098 (size=4875) 2024-11-24T01:46:45,407 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,411 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T01:46:45,411 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T01:46:45,411 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,415 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T01:46:45,415 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T01:46:45,415 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,419 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T01:46:45,419 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T01:46:45,419 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a386e431248dccab721bfd7e385c379c 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T01:46:45,436 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp/a/5feb0a56dce949058a293aab369d9a15 is 55, key is testCompactedBulkLoadedFiles/a:a/1732412804972/Put/seqid=0 2024-11-24T01:46:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741921_1099 (size=5107) 2024-11-24T01:46:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741921_1099 (size=5107) 2024-11-24T01:46:45,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741921_1099 (size=5107) 2024-11-24T01:46:45,443 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp/a/5feb0a56dce949058a293aab369d9a15 2024-11-24T01:46:45,449 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp/a/5feb0a56dce949058a293aab369d9a15 as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15 2024-11-24T01:46:45,454 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T01:46:45,455 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for a386e431248dccab721bfd7e385c379c in 36ms, sequenceid=4, compaction requested=false 2024-11-24T01:46:45,455 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a386e431248dccab721bfd7e385c379c: 2024-11-24T01:46:45,456 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ 2024-11-24T01:46:45,457 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ 2024-11-24T01:46:45,458 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ 2024-11-24T01:46:45,459 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile0 into a386e431248dccab721bfd7e385c379c/a as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ - updating store file list. 2024-11-24T01:46:45,464 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d9e3a291af43438ca46696ea99764561_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:45,464 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ into a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,464 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile0 into a386e431248dccab721bfd7e385c379c/a (new location: hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_) 2024-11-24T01:46:45,465 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile1 into a386e431248dccab721bfd7e385c379c/a as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ - updating store file list. 2024-11-24T01:46:45,470 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 721d990a163f454394546f4202d922cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:45,470 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ into a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,470 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile1 into a386e431248dccab721bfd7e385c379c/a (new location: hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_) 2024-11-24T01:46:45,472 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile2 into a386e431248dccab721bfd7e385c379c/a as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ - updating store file list. 2024-11-24T01:46:45,476 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a75efc1b9ec42b783e60bab448223e5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:45,476 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ into a386e431248dccab721bfd7e385c379c/a 2024-11-24T01:46:45,476 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:34675/hbase/testCompactedBulkLoadedFiles/hfile2 into a386e431248dccab721bfd7e385c379c/a (new location: hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_) 2024-11-24T01:46:45,485 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T01:46:45,485 DEBUG [Time-limited test {}] regionserver.HStore(1541): a386e431248dccab721bfd7e385c379c/a is initiating major compaction (all files) 2024-11-24T01:46:45,485 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a386e431248dccab721bfd7e385c379c/a in testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:45,486 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_] into tmpdir=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp, totalSize=19.3 K 2024-11-24T01:46:45,486 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5feb0a56dce949058a293aab369d9a15, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732412804972 2024-11-24T01:46:45,487 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d9e3a291af43438ca46696ea99764561_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T01:46:45,487 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 721d990a163f454394546f4202d922cf_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T01:46:45,488 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5a75efc1b9ec42b783e60bab448223e5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T01:46:45,500 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp/a/5a6271bb966f4b37bf993904309fe30d is 55, key is testCompactedBulkLoadedFiles/a:a/1732412804972/Put/seqid=0 2024-11-24T01:46:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741922_1100 (size=6154) 2024-11-24T01:46:45,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741922_1100 (size=6154) 2024-11-24T01:46:45,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741922_1100 (size=6154) 2024-11-24T01:46:45,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/.tmp/a/5a6271bb966f4b37bf993904309fe30d as hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a6271bb966f4b37bf993904309fe30d 2024-11-24T01:46:45,518 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in a386e431248dccab721bfd7e385c379c/a of a386e431248dccab721bfd7e385c379c into 5a6271bb966f4b37bf993904309fe30d(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T01:46:45,518 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a386e431248dccab721bfd7e385c379c: 2024-11-24T01:46:45,519 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T01:46:45,519 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T01:46:45,565 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, size=0 (0bytes) 2024-11-24T01:46:45,565 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 might be still open, length is 0 2024-11-24T01:46:45,565 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 2024-11-24T01:46:45,566 WARN [IPC Server handler 1 on default port 34675 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T01:46:45,566 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 after 1ms 2024-11-24T01:46:46,019 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:51954 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39823:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51954 dst: /127.0.0.1:39823 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39823 remote=/127.0.0.1:51954]. Total timeout mills is 60000, 59499 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:46,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60150 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33251:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60150 dst: /127.0.0.1:33251 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:46,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34764 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34764 dst: /127.0.0.1:39381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:46,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741917_1101 (size=1794) 2024-11-24T01:46:46,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741917_1101 (size=1794) 2024-11-24T01:46:49,174 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T01:46:49,567 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 after 4002ms 2024-11-24T01:46:49,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:49,599 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 took 4034ms 2024-11-24T01:46:49,601 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908; continuing. 2024-11-24T01:46:49,601 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 so closing down 2024-11-24T01:46:49,601 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:49,602 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732412804908.temp 2024-11-24T01:46:49,627 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp 2024-11-24T01:46:49,627 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741923_1102 (size=800) 2024-11-24T01:46:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741923_1102 (size=800) 2024-11-24T01:46:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741923_1102 (size=800) 2024-11-24T01:46:49,633 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:49,635 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp to hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 2024-11-24T01:46:49,635 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T01:46:49,635 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, journal: Splitting hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, size=0 (0bytes) at 1732412805565Finishing writing output for hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 so closing down at 1732412809601 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp at 1732412809627 (+26 ms)3 split writer threads finished at 1732412809627Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732412809633 (+6 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000003-wal.1732412804908.temp to hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 at 1732412809635 (+2 ms)Processed 5 edits across 1 Regions in 35 ms; skipped=3; WAL=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908, size=0, length=0, corrupted=false, cancelled=false at 1732412809635 2024-11-24T01:46:49,637 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412804908 2024-11-24T01:46:49,637 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 2024-11-24T01:46:49,638 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:49,639 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:49,653 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412809640, exclude list is [], retry=0 2024-11-24T01:46:49,655 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:49,655 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:49,656 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:49,678 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412809640 2024-11-24T01:46:49,679 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:49,679 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a386e431248dccab721bfd7e385c379c, NAME => 'testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:49,679 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:49,679 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,679 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,680 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,681 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName a 2024-11-24T01:46:49,681 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,688 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a6271bb966f4b37bf993904309fe30d 2024-11-24T01:46:49,692 DEBUG [StoreFileOpener-a386e431248dccab721bfd7e385c379c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5a75efc1b9ec42b783e60bab448223e5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:49,692 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ 2024-11-24T01:46:49,696 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15 2024-11-24T01:46:49,700 DEBUG [StoreFileOpener-a386e431248dccab721bfd7e385c379c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 721d990a163f454394546f4202d922cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:49,700 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ 2024-11-24T01:46:49,704 DEBUG [StoreFileOpener-a386e431248dccab721bfd7e385c379c-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d9e3a291af43438ca46696ea99764561_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T01:46:49,704 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ 2024-11-24T01:46:49,704 WARN [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6d4b8948 2024-11-24T01:46:49,704 WARN [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6d4b8948 2024-11-24T01:46:49,704 WARN [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6d4b8948 2024-11-24T01:46:49,704 WARN [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@6d4b8948 2024-11-24T01:46:49,705 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_] to archive 2024-11-24T01:46:49,706 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T01:46:49,708 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ to hdfs://localhost:34675/hbase/archive/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5a75efc1b9ec42b783e60bab448223e5_SeqId_4_ 2024-11-24T01:46:49,710 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15 to hdfs://localhost:34675/hbase/archive/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/5feb0a56dce949058a293aab369d9a15 2024-11-24T01:46:49,711 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ to hdfs://localhost:34675/hbase/archive/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/721d990a163f454394546f4202d922cf_SeqId_4_ 2024-11-24T01:46:49,712 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ to hdfs://localhost:34675/hbase/archive/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/a/d9e3a291af43438ca46696ea99764561_SeqId_4_ 2024-11-24T01:46:49,712 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,713 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,713 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName b 2024-11-24T01:46:49,713 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,714 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,714 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,714 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a386e431248dccab721bfd7e385c379c columnFamilyName c 2024-11-24T01:46:49,714 DEBUG [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,715 INFO [StoreOpener-a386e431248dccab721bfd7e385c379c-1 {}] regionserver.HStore(327): Store=a386e431248dccab721bfd7e385c379c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,717 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,718 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 2024-11-24T01:46:49,741 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:49,743 DEBUG [Time-limited test {}] regionserver.HRegion(5836): a386e431248dccab721bfd7e385c379c : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "a386e431248dccab721bfd7e385c379c" family_name: "a" compaction_input: "5feb0a56dce949058a293aab369d9a15" compaction_input: "d9e3a291af43438ca46696ea99764561_SeqId_4_" compaction_input: "721d990a163f454394546f4202d922cf_SeqId_4_" compaction_input: "5a75efc1b9ec42b783e60bab448223e5_SeqId_4_" compaction_output: "5a6271bb966f4b37bf993904309fe30d" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T01:46:49,743 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T01:46:49,743 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 2024-11-24T01:46:49,744 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/0000000000000000008 2024-11-24T01:46:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,745 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,746 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:49,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a386e431248dccab721bfd7e385c379c 2024-11-24T01:46:49,749 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testCompactedBulkLoadedFiles/a386e431248dccab721bfd7e385c379c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T01:46:49,750 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a386e431248dccab721bfd7e385c379c; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73778346, jitterRate=0.09938302636146545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:49,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a386e431248dccab721bfd7e385c379c: Writing region info on filesystem at 1732412809679Initializing all the Stores at 1732412809680 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809680Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809680Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809680Cleaning up temporary data from old regions at 1732412809745 (+65 ms)Region opened successfully at 1732412809750 (+5 ms) 2024-11-24T01:46:49,752 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a386e431248dccab721bfd7e385c379c, disabling compactions & flushes 2024-11-24T01:46:49,752 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:49,752 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:49,752 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. after waiting 0 ms 2024-11-24T01:46:49,752 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:49,753 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732412804730.a386e431248dccab721bfd7e385c379c. 2024-11-24T01:46:49,753 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a386e431248dccab721bfd7e385c379c: Waiting for close lock at 1732412809752Disabling compacts and flushes for region at 1732412809752Disabling writes for close at 1732412809752Writing region close event to WAL at 1732412809753 (+1 ms)Closed at 1732412809753 2024-11-24T01:46:49,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741924_1103 (size=175) 2024-11-24T01:46:49,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741924_1103 (size=175) 2024-11-24T01:46:49,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741924_1103 (size=175) 2024-11-24T01:46:49,757 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:49,757 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732412809640) 2024-11-24T01:46:49,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 429) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42853 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:34675 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:42853 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-694865083_22 at /127.0.0.1:52016 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-694865083_22 at /127.0.0.1:34824 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-694865083_22 at /127.0.0.1:60228 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34675 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1260) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=206 (was 224), ProcessCount=11 (was 11), AvailableMemoryMB=7360 (was 7372) 2024-11-24T01:46:49,771 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-24T01:46:49,785 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=206, ProcessCount=11, AvailableMemoryMB=7360 2024-11-24T01:46:49,785 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-24T01:46:49,804 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:49,805 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T01:46:49,805 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T01:46:49,808 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82504350, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/hregion-82504350, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:49,819 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82504350/hregion-82504350.1732412809808, exclude list is [], retry=0 2024-11-24T01:46:49,821 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:49,821 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:49,821 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:49,845 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82504350/hregion-82504350.1732412809808 2024-11-24T01:46:49,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873)] 2024-11-24T01:46:49,845 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 895d3a364b388d17f40b7ca7c85efe2c, NAME => 'testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34675/hbase 2024-11-24T01:46:49,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741926_1105 (size=67) 2024-11-24T01:46:49,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741926_1105 (size=67) 2024-11-24T01:46:49,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741926_1105 (size=67) 2024-11-24T01:46:49,853 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:49,854 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,855 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName a 2024-11-24T01:46:49,855 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,855 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,855 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,856 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName b 2024-11-24T01:46:49,856 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,857 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,857 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,858 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName c 2024-11-24T01:46:49,858 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,858 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,858 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,859 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,860 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,860 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,861 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:49,862 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,864 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T01:46:49,865 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895d3a364b388d17f40b7ca7c85efe2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60764804, jitterRate=-0.09453386068344116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895d3a364b388d17f40b7ca7c85efe2c: Writing region info on filesystem at 1732412809853Initializing all the Stores at 1732412809853Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809853Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809854 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809854Cleaning up temporary data from old regions at 1732412809860 (+6 ms)Region opened successfully at 1732412809865 (+5 ms) 2024-11-24T01:46:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895d3a364b388d17f40b7ca7c85efe2c, disabling compactions & flushes 2024-11-24T01:46:49,865 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. after waiting 0 ms 2024-11-24T01:46:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,866 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,866 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895d3a364b388d17f40b7ca7c85efe2c: Waiting for close lock at 1732412809865Disabling compacts and flushes for region at 1732412809865Disabling writes for close at 1732412809865Writing region close event to WAL at 1732412809866 (+1 ms)Closed at 1732412809866 2024-11-24T01:46:49,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741925_1104 (size=175) 2024-11-24T01:46:49,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741925_1104 (size=175) 2024-11-24T01:46:49,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741925_1104 (size=175) 2024-11-24T01:46:49,870 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:49,870 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82504350:(num 1732412809808) 2024-11-24T01:46:49,870 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:49,872 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:49,885 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, exclude list is [], retry=0 2024-11-24T01:46:49,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:49,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:49,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:49,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 2024-11-24T01:46:49,911 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:49,911 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 895d3a364b388d17f40b7ca7c85efe2c, NAME => 'testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:49,911 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:49,911 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,911 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,913 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,913 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName a 2024-11-24T01:46:49,914 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,914 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,914 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,915 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName b 2024-11-24T01:46:49,915 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,915 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,915 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,916 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName c 2024-11-24T01:46:49,916 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:49,916 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:49,916 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,917 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,918 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,919 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:49,921 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:49,922 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895d3a364b388d17f40b7ca7c85efe2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60884564, jitterRate=-0.09274929761886597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:49,922 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895d3a364b388d17f40b7ca7c85efe2c: Writing region info on filesystem at 1732412809912Initializing all the Stores at 1732412809912Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809913 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809913Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412809913Cleaning up temporary data from old regions at 1732412809919 (+6 ms)Region opened successfully at 1732412809922 (+3 ms) 2024-11-24T01:46:49,930 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895d3a364b388d17f40b7ca7c85efe2c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T01:46:49,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/2dceb2ab537348e88b5357df1c55aa6c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732412809922/Put/seqid=0 2024-11-24T01:46:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741928_1107 (size=5958) 2024-11-24T01:46:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741928_1107 (size=5958) 2024-11-24T01:46:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741928_1107 (size=5958) 2024-11-24T01:46:49,950 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/2dceb2ab537348e88b5357df1c55aa6c 2024-11-24T01:46:49,955 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/2dceb2ab537348e88b5357df1c55aa6c as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/2dceb2ab537348e88b5357df1c55aa6c 2024-11-24T01:46:49,960 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/2dceb2ab537348e88b5357df1c55aa6c, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T01:46:49,961 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 895d3a364b388d17f40b7ca7c85efe2c in 32ms, sequenceid=13, compaction requested=false 2024-11-24T01:46:49,961 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 895d3a364b388d17f40b7ca7c85efe2c: 2024-11-24T01:46:49,975 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895d3a364b388d17f40b7ca7c85efe2c, disabling compactions & flushes 2024-11-24T01:46:49,975 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,975 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,975 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. after waiting 0 ms 2024-11-24T01:46:49,975 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,976 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,976 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:49,976 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895d3a364b388d17f40b7ca7c85efe2c: Waiting for close lock at 1732412809975Disabling compacts and flushes for region at 1732412809975Disabling writes for close at 1732412809975Writing region close event to WAL at 1732412809976 (+1 ms)Closed at 1732412809976 2024-11-24T01:46:49,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741927_1106 (size=6243) 2024-11-24T01:46:49,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741927_1106 (size=6243) 2024-11-24T01:46:49,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741927_1106 (size=6243) 2024-11-24T01:46:49,992 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, size=6.1 K (6243bytes) 2024-11-24T01:46:49,992 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 2024-11-24T01:46:49,993 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 after 1ms 2024-11-24T01:46:50,025 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:50,025 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 took 33ms 2024-11-24T01:46:50,027 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 so closing down 2024-11-24T01:46:50,027 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:50,028 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732412809873.temp 2024-11-24T01:46:50,050 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp 2024-11-24T01:46:50,051 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:50,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741929_1108 (size=5575) 2024-11-24T01:46:50,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741929_1108 (size=5575) 2024-11-24T01:46:50,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741929_1108 (size=5575) 2024-11-24T01:46:50,058 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:50,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 2024-11-24T01:46:50,059 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-24T01:46:50,059 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, journal: Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, size=6.1 K (6243bytes) at 1732412809992Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 so closing down at 1732412810027 (+35 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp at 1732412810051 (+24 ms)3 split writer threads finished at 1732412810051Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732412810058 (+7 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000003-wal.1732412809873.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 at 1732412810059 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732412810059 2024-11-24T01:46:50,061 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412809873 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412809873 2024-11-24T01:46:50,062 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 2024-11-24T01:46:50,062 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:50,064 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:50,076 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, exclude list is [], retry=0 2024-11-24T01:46:50,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:50,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:50,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:50,104 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 2024-11-24T01:46:50,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33015:33015),(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645)] 2024-11-24T01:46:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 895d3a364b388d17f40b7ca7c85efe2c, NAME => 'testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.', STARTKEY => '', ENDKEY => ''} 2024-11-24T01:46:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,104 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,106 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,106 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName a 2024-11-24T01:46:50,107 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:50,111 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/2dceb2ab537348e88b5357df1c55aa6c 2024-11-24T01:46:50,111 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:50,112 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,112 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName b 2024-11-24T01:46:50,112 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:50,113 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:50,113 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,113 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName c 2024-11-24T01:46:50,113 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:50,114 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:50,114 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,116 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,116 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 2024-11-24T01:46:50,142 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:50,143 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 2024-11-24T01:46:50,143 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895d3a364b388d17f40b7ca7c85efe2c 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T01:46:50,156 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/66601da98a164bcab0c58cce48e9e440 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732412809961/Put/seqid=0 2024-11-24T01:46:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741931_1110 (size=5958) 2024-11-24T01:46:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741931_1110 (size=5958) 2024-11-24T01:46:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741931_1110 (size=5958) 2024-11-24T01:46:50,163 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/66601da98a164bcab0c58cce48e9e440 2024-11-24T01:46:50,182 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/131fed9ba1ef429ba0abe2d2b75e9626 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732412809967/Put/seqid=0 2024-11-24T01:46:50,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741932_1111 (size=5958) 2024-11-24T01:46:50,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741932_1111 (size=5958) 2024-11-24T01:46:50,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741932_1111 (size=5958) 2024-11-24T01:46:50,188 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/131fed9ba1ef429ba0abe2d2b75e9626 2024-11-24T01:46:50,193 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/66601da98a164bcab0c58cce48e9e440 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/b/66601da98a164bcab0c58cce48e9e440 2024-11-24T01:46:50,198 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/b/66601da98a164bcab0c58cce48e9e440, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T01:46:50,199 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/131fed9ba1ef429ba0abe2d2b75e9626 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/c/131fed9ba1ef429ba0abe2d2b75e9626 2024-11-24T01:46:50,204 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/c/131fed9ba1ef429ba0abe2d2b75e9626, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T01:46:50,204 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 895d3a364b388d17f40b7ca7c85efe2c in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T01:46:50,205 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000035 2024-11-24T01:46:50,206 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,206 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,207 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:50,208 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:50,210 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T01:46:50,211 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895d3a364b388d17f40b7ca7c85efe2c; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65020337, jitterRate=-0.031121477484703064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:50,211 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895d3a364b388d17f40b7ca7c85efe2c: Writing region info on filesystem at 1732412810105Initializing all the Stores at 1732412810105Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412810105Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412810106 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412810106Obtaining lock to block concurrent updates at 1732412810143 (+37 ms)Preparing flush snapshotting stores in 895d3a364b388d17f40b7ca7c85efe2c at 1732412810143Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732412810143Flushing stores of testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. at 1732412810143Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: creating writer at 1732412810143Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: appending metadata at 1732412810156 (+13 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: closing flushed file at 1732412810156Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: creating writer at 1732412810167 (+11 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: appending metadata at 1732412810181 (+14 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: closing flushed file at 1732412810181Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4538f181: reopening flushed file at 1732412810192 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4357241b: reopening flushed file at 1732412810198 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 895d3a364b388d17f40b7ca7c85efe2c in 61ms, sequenceid=35, compaction requested=false; wal=null at 1732412810204 (+6 ms)Cleaning up temporary data from old regions at 1732412810206 (+2 ms)Region opened successfully at 1732412810211 (+5 ms) 2024-11-24T01:46:50,273 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, size=0 (0bytes) 2024-11-24T01:46:50,273 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 might be still open, length is 0 2024-11-24T01:46:50,273 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 2024-11-24T01:46:50,273 WARN [IPC Server handler 0 on default port 34675 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T01:46:50,274 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 after 1ms 2024-11-24T01:46:52,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:52084 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39823:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52084 dst: /127.0.0.1:39823 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39823 remote=/127.0.0.1:52084]. Total timeout mills is 60000, 58219 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:52,021 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:34890 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39381:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34890 dst: /127.0.0.1:39381 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:52,021 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1410427140_22 at /127.0.0.1:60282 [Receiving block BP-2032936213-172.17.0.2-1732412770132:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33251:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60282 dst: /127.0.0.1:33251 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:52,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741930_1112 (size=5567) 2024-11-24T01:46:52,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741930_1112 (size=5567) 2024-11-24T01:46:54,275 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 after 4002ms 2024-11-24T01:46:54,291 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T01:46:54,291 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:54,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T01:46:54,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T01:46:54,310 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:54,310 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 took 4037ms 2024-11-24T01:46:54,313 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064; continuing. 2024-11-24T01:46:54,313 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 so closing down 2024-11-24T01:46:54,313 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T01:46:54,314 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732412810064.temp 2024-11-24T01:46:54,337 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp 2024-11-24T01:46:54,338 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T01:46:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741933_1113 (size=5575) 2024-11-24T01:46:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741933_1113 (size=5575) 2024-11-24T01:46:54,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741933_1113 (size=5575) 2024-11-24T01:46:54,344 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T01:46:54,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 2024-11-24T01:46:54,345 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T01:46:54,345 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, journal: Splitting hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, size=0 (0bytes) at 1732412810273Finishing writing output for hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 so closing down at 1732412814313 (+4040 ms)Creating recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp at 1732412814337 (+24 ms)3 split writer threads finished at 1732412814338 (+1 ms)Closed recovered edits writer path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732412814344 (+6 ms)Rename recovered edits hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000037-wal.1732412810064.temp to hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 at 1732412814345 (+1 ms)Processed 30 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064, size=0, length=0, corrupted=false, cancelled=false at 1732412814345 2024-11-24T01:46:54,347 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 to hdfs://localhost:34675/hbase/oldWALs/wal.1732412810064 2024-11-24T01:46:54,347 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 2024-11-24T01:46:54,348 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T01:46:54,349 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:34675/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803, archiveDir=hdfs://localhost:34675/hbase/oldWALs, maxLogs=32 2024-11-24T01:46:54,362 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412814350, exclude list is [], retry=0 2024-11-24T01:46:54,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39381,DS-d62473c1-5a99-42df-bfc9-84a75470c6fd,DISK] 2024-11-24T01:46:54,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33251,DS-fa9580b4-2608-49b3-95a9-6ce69d8619e1,DISK] 2024-11-24T01:46:54,365 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39823,DS-086d5bc5-16c3-476a-8d39-9b25c17950c7,DISK] 2024-11-24T01:46:54,387 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412814350 2024-11-24T01:46:54,387 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38873:38873),(127.0.0.1/127.0.0.1:38645:38645),(127.0.0.1/127.0.0.1:33015:33015)] 2024-11-24T01:46:54,388 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T01:46:54,389 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,390 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName a 2024-11-24T01:46:54,390 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:54,394 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/2dceb2ab537348e88b5357df1c55aa6c 2024-11-24T01:46:54,394 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:54,394 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,395 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName b 2024-11-24T01:46:54,395 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:54,400 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/b/66601da98a164bcab0c58cce48e9e440 2024-11-24T01:46:54,400 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:54,400 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,401 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895d3a364b388d17f40b7ca7c85efe2c columnFamilyName c 2024-11-24T01:46:54,401 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T01:46:54,406 DEBUG [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/c/131fed9ba1ef429ba0abe2d2b75e9626 2024-11-24T01:46:54,406 INFO [StoreOpener-895d3a364b388d17f40b7ca7c85efe2c-1 {}] regionserver.HStore(327): Store=895d3a364b388d17f40b7ca7c85efe2c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T01:46:54,406 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,407 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,409 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 2024-11-24T01:46:54,434 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T01:46:54,437 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 2024-11-24T01:46:54,437 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895d3a364b388d17f40b7ca7c85efe2c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T01:46:54,450 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/4a79368c54f6488d9d120aac913c018a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732412810218/Put/seqid=0 2024-11-24T01:46:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741935_1115 (size=5958) 2024-11-24T01:46:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741935_1115 (size=5958) 2024-11-24T01:46:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741935_1115 (size=5958) 2024-11-24T01:46:54,456 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/4a79368c54f6488d9d120aac913c018a 2024-11-24T01:46:54,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/cf5a0455fd9a4f7591974c9014e32684 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732412810226/Put/seqid=0 2024-11-24T01:46:54,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741936_1116 (size=5958) 2024-11-24T01:46:54,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741936_1116 (size=5958) 2024-11-24T01:46:54,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741936_1116 (size=5958) 2024-11-24T01:46:54,478 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/cf5a0455fd9a4f7591974c9014e32684 2024-11-24T01:46:54,496 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/b14f427e0a2f4bdbbd0bd650d97f3f65 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732412810232/Put/seqid=0 2024-11-24T01:46:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741937_1117 (size=5958) 2024-11-24T01:46:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741937_1117 (size=5958) 2024-11-24T01:46:54,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741937_1117 (size=5958) 2024-11-24T01:46:54,503 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/b14f427e0a2f4bdbbd0bd650d97f3f65 2024-11-24T01:46:54,508 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/a/4a79368c54f6488d9d120aac913c018a as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/4a79368c54f6488d9d120aac913c018a 2024-11-24T01:46:54,513 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/a/4a79368c54f6488d9d120aac913c018a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T01:46:54,514 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/b/cf5a0455fd9a4f7591974c9014e32684 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/b/cf5a0455fd9a4f7591974c9014e32684 2024-11-24T01:46:54,518 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/b/cf5a0455fd9a4f7591974c9014e32684, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T01:46:54,519 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/.tmp/c/b14f427e0a2f4bdbbd0bd650d97f3f65 as hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/c/b14f427e0a2f4bdbbd0bd650d97f3f65 2024-11-24T01:46:54,524 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/c/b14f427e0a2f4bdbbd0bd650d97f3f65, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T01:46:54,524 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 895d3a364b388d17f40b7ca7c85efe2c in 87ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T01:46:54,525 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/0000000000000000066 2024-11-24T01:46:54,526 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,526 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,526 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T01:46:54,527 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895d3a364b388d17f40b7ca7c85efe2c 2024-11-24T01:46:54,529 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/hbase/data/default/testReplayEditsWrittenViaHRegion/895d3a364b388d17f40b7ca7c85efe2c/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T01:46:54,530 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895d3a364b388d17f40b7ca7c85efe2c; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65938317, jitterRate=-0.017442509531974792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T01:46:54,530 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895d3a364b388d17f40b7ca7c85efe2c: Writing region info on filesystem at 1732412814388Initializing all the Stores at 1732412814389 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412814389Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412814389Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732412814389Obtaining lock to block concurrent updates at 1732412814437 (+48 ms)Preparing flush snapshotting stores in 895d3a364b388d17f40b7ca7c85efe2c at 1732412814437Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732412814437Flushing stores of testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. at 1732412814437Flushing 895d3a364b388d17f40b7ca7c85efe2c/a: creating writer at 1732412814437Flushing 895d3a364b388d17f40b7ca7c85efe2c/a: appending metadata at 1732412814449 (+12 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/a: closing flushed file at 1732412814449Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: creating writer at 1732412814459 (+10 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: appending metadata at 1732412814472 (+13 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/b: closing flushed file at 1732412814472Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: creating writer at 1732412814482 (+10 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: appending metadata at 1732412814496 (+14 ms)Flushing 895d3a364b388d17f40b7ca7c85efe2c/c: closing flushed file at 1732412814496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195ecc2e: reopening flushed file at 1732412814507 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52831f7d: reopening flushed file at 1732412814513 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e3e092: reopening flushed file at 1732412814519 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 895d3a364b388d17f40b7ca7c85efe2c in 87ms, sequenceid=66, compaction requested=false; wal=null at 1732412814524 (+5 ms)Cleaning up temporary data from old regions at 1732412814526 (+2 ms)Region opened successfully at 1732412814530 (+4 ms) 2024-11-24T01:46:54,540 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895d3a364b388d17f40b7ca7c85efe2c, disabling compactions & flushes 2024-11-24T01:46:54,540 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:54,540 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:54,540 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. after waiting 0 ms 2024-11-24T01:46:54,540 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:54,542 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732412809804.895d3a364b388d17f40b7ca7c85efe2c. 2024-11-24T01:46:54,542 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895d3a364b388d17f40b7ca7c85efe2c: Waiting for close lock at 1732412814540Disabling compacts and flushes for region at 1732412814540Disabling writes for close at 1732412814540Writing region close event to WAL at 1732412814542 (+2 ms)Closed at 1732412814542 2024-11-24T01:46:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741934_1114 (size=175) 2024-11-24T01:46:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741934_1114 (size=175) 2024-11-24T01:46:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741934_1114 (size=175) 2024-11-24T01:46:54,545 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T01:46:54,545 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732412814350) 2024-11-24T01:46:54,558 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=447 (was 443) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_71447977_22 at /127.0.0.1:40858 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_71447977_22 at /127.0.0.1:59998 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:34675 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:34675 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_71447977_22 at /127.0.0.1:56762 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1408 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=205 (was 206), ProcessCount=11 (was 11), AvailableMemoryMB=7351 (was 7360) 2024-11-24T01:46:54,559 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1408 is superior to 1024 2024-11-24T01:46:54,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T01:46:54,559 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T01:46:54,559 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T01:46:54,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T01:46:54,560 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T01:46:54,560 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=826943530, stopped=false 2024-11-24T01:46:54,560 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a67e7d17e593,37581,1732412773942 2024-11-24T01:46:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T01:46:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:54,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:54,608 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T01:46:54,608 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T01:46:54,608 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T01:46:54,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:54,608 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a67e7d17e593,33283,1732412774831' ***** 2024-11-24T01:46:54,608 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T01:46:54,608 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a67e7d17e593,43635,1732412774891' ***** 2024-11-24T01:46:54,608 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T01:46:54,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:54,609 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T01:46:54,609 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T01:46:54,609 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T01:46:54,609 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bd22b2d59c54a42edcaf121e896c37c, disabling compactions & flushes 2024-11-24T01:46:54,609 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(959): stopping server a67e7d17e593,43635,1732412774891 2024-11-24T01:46:54,609 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T01:46:54,609 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. after waiting 0 ms 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(959): stopping server a67e7d17e593,33283,1732412774831 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a67e7d17e593:43635. 2024-11-24T01:46:54,609 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T01:46:54,609 DEBUG [RS:2;a67e7d17e593:43635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a67e7d17e593:33283. 2024-11-24T01:46:54,609 DEBUG [RS:2;a67e7d17e593:43635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,609 DEBUG [RS:1;a67e7d17e593:33283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T01:46:54,609 DEBUG [RS:1;a67e7d17e593:33283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,609 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(976): stopping server a67e7d17e593,43635,1732412774891; all regions closed. 2024-11-24T01:46:54,609 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T01:46:54,610 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T01:46:54,610 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T01:46:54,610 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T01:46:54,610 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T01:46:54,610 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 5bd22b2d59c54a42edcaf121e896c37c=testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c.} 2024-11-24T01:46:54,610 DEBUG [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5bd22b2d59c54a42edcaf121e896c37c 2024-11-24T01:46:54,610 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T01:46:54,610 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T01:46:54,610 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T01:46:54,610 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T01:46:54,610 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T01:46:54,610 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T01:46:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741833_1009 (size=175) 2024-11-24T01:46:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741833_1009 (size=175) 2024-11-24T01:46:54,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741833_1009 (size=175) 2024-11-24T01:46:54,613 DEBUG [RS:2;a67e7d17e593:43635 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C43635%2C1732412774891:(num 1732412776506) 2024-11-24T01:46:54,614 DEBUG [RS:2;a67e7d17e593:43635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T01:46:54,614 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/default/testReplayEditsAfterRegionMovedWithMultiCF/5bd22b2d59c54a42edcaf121e896c37c/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] hbase.ChoreService(370): Chore service for: regionserver/a67e7d17e593:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T01:46:54,614 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T01:46:54,614 INFO [RS:2;a67e7d17e593:43635 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43635 2024-11-24T01:46:54,615 INFO [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:54,615 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bd22b2d59c54a42edcaf121e896c37c: Waiting for close lock at 1732412814609Running coprocessor pre-close hooks at 1732412814609Disabling compacts and flushes for region at 1732412814609Disabling writes for close at 1732412814609Writing region close event to WAL at 1732412814610 (+1 ms)Running coprocessor post-close hooks at 1732412814615 (+5 ms)Closed at 1732412814615 2024-11-24T01:46:54,615 DEBUG [RS_CLOSE_REGION-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c. 2024-11-24T01:46:54,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T01:46:54,624 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a67e7d17e593,43635,1732412774891 2024-11-24T01:46:54,624 INFO [RS:2;a67e7d17e593:43635 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T01:46:54,630 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/info/6b95a029edcd44c390fa6eb9aef0d91e is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732412790368.5bd22b2d59c54a42edcaf121e896c37c./info:regioninfo/1732412793599/Put/seqid=0 2024-11-24T01:46:54,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a67e7d17e593,43635,1732412774891] 2024-11-24T01:46:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741938_1118 (size=8243) 2024-11-24T01:46:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741938_1118 (size=8243) 2024-11-24T01:46:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741938_1118 (size=8243) 2024-11-24T01:46:54,637 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/info/6b95a029edcd44c390fa6eb9aef0d91e 2024-11-24T01:46:54,640 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a67e7d17e593,43635,1732412774891 already deleted, retry=false 2024-11-24T01:46:54,641 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a67e7d17e593,43635,1732412774891 expired; onlineServers=1 2024-11-24T01:46:54,660 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/ns/b4e3a7b01ad1478da811f0339859b16a is 43, key is default/ns:d/1732412777304/Put/seqid=0 2024-11-24T01:46:54,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741939_1119 (size=5153) 2024-11-24T01:46:54,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741939_1119 (size=5153) 2024-11-24T01:46:54,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741939_1119 (size=5153) 2024-11-24T01:46:54,666 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/ns/b4e3a7b01ad1478da811f0339859b16a 2024-11-24T01:46:54,684 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/table/56d5b223d94d46e1b4769564c94b0e7c is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732412790783/Put/seqid=0 2024-11-24T01:46:54,684 INFO [regionserver/a67e7d17e593:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:54,687 INFO [regionserver/a67e7d17e593:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:54,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741940_1120 (size=5431) 2024-11-24T01:46:54,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741940_1120 (size=5431) 2024-11-24T01:46:54,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741940_1120 (size=5431) 2024-11-24T01:46:54,692 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/table/56d5b223d94d46e1b4769564c94b0e7c 2024-11-24T01:46:54,697 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/info/6b95a029edcd44c390fa6eb9aef0d91e as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/info/6b95a029edcd44c390fa6eb9aef0d91e 2024-11-24T01:46:54,703 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/info/6b95a029edcd44c390fa6eb9aef0d91e, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T01:46:54,704 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/ns/b4e3a7b01ad1478da811f0339859b16a as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/ns/b4e3a7b01ad1478da811f0339859b16a 2024-11-24T01:46:54,709 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/ns/b4e3a7b01ad1478da811f0339859b16a, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T01:46:54,710 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/.tmp/table/56d5b223d94d46e1b4769564c94b0e7c as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/table/56d5b223d94d46e1b4769564c94b0e7c 2024-11-24T01:46:54,714 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/table/56d5b223d94d46e1b4769564c94b0e7c, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T01:46:54,715 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false 2024-11-24T01:46:54,720 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T01:46:54,720 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T01:46:54,720 INFO [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T01:46:54,720 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732412814610Running coprocessor pre-close hooks at 1732412814610Disabling compacts and flushes for region at 1732412814610Disabling writes for close at 1732412814610Obtaining lock to block concurrent updates at 1732412814610Preparing flush snapshotting stores in 1588230740 at 1732412814610Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732412814611 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732412814612 (+1 ms)Flushing 1588230740/info: creating writer at 1732412814612Flushing 1588230740/info: appending metadata at 1732412814630 (+18 ms)Flushing 1588230740/info: closing flushed file at 1732412814630Flushing 1588230740/ns: creating writer at 1732412814643 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732412814660 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732412814660Flushing 1588230740/table: creating writer at 1732412814671 (+11 ms)Flushing 1588230740/table: appending metadata at 1732412814683 (+12 ms)Flushing 1588230740/table: closing flushed file at 1732412814683Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52539ce0: reopening flushed file at 1732412814696 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10957966: reopening flushed file at 1732412814703 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5750d053: reopening flushed file at 1732412814709 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false at 1732412814715 (+6 ms)Writing region close event to WAL at 1732412814717 (+2 ms)Running coprocessor post-close hooks at 1732412814720 (+3 ms)Closed at 1732412814720 2024-11-24T01:46:54,720 DEBUG [RS_CLOSE_META-regionserver/a67e7d17e593:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T01:46:54,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:54,732 INFO [RS:2;a67e7d17e593:43635 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T01:46:54,732 INFO [RS:2;a67e7d17e593:43635 {}] regionserver.HRegionServer(1031): Exiting; stopping=a67e7d17e593,43635,1732412774891; zookeeper connection closed. 2024-11-24T01:46:54,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x1016bcea5fe0003, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:54,733 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@118eb1ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@118eb1ac 2024-11-24T01:46:54,810 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(976): stopping server a67e7d17e593,33283,1732412774831; all regions closed. 2024-11-24T01:46:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741836_1012 (size=9487) 2024-11-24T01:46:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741836_1012 (size=9487) 2024-11-24T01:46:54,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741836_1012 (size=9487) 2024-11-24T01:46:54,821 DEBUG [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs 2024-11-24T01:46:54,821 INFO [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C33283%2C1732412774831.meta:.meta(num 1732412777094) 2024-11-24T01:46:54,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741835_1011 (size=3621) 2024-11-24T01:46:54,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741835_1011 (size=3621) 2024-11-24T01:46:54,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741835_1011 (size=3621) 2024-11-24T01:46:54,825 DEBUG [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/oldWALs 2024-11-24T01:46:54,825 INFO [RS:1;a67e7d17e593:33283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a67e7d17e593%2C33283%2C1732412774831:(num 1732412776506) 2024-11-24T01:46:54,825 DEBUG [RS:1;a67e7d17e593:33283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T01:46:54,825 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T01:46:54,826 INFO [RS:1;a67e7d17e593:33283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T01:46:54,826 INFO [RS:1;a67e7d17e593:33283 {}] hbase.ChoreService(370): Chore service for: regionserver/a67e7d17e593:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T01:46:54,826 INFO [RS:1;a67e7d17e593:33283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T01:46:54,826 INFO [regionserver/a67e7d17e593:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T01:46:54,826 INFO [RS:1;a67e7d17e593:33283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33283 2024-11-24T01:46:54,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T01:46:54,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a67e7d17e593,33283,1732412774831 2024-11-24T01:46:54,849 INFO [RS:1;a67e7d17e593:33283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T01:46:54,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a67e7d17e593,33283,1732412774831] 2024-11-24T01:46:54,866 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a67e7d17e593,33283,1732412774831 already deleted, retry=false 2024-11-24T01:46:54,866 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a67e7d17e593,33283,1732412774831 expired; onlineServers=0 2024-11-24T01:46:54,866 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a67e7d17e593,37581,1732412773942' ***** 2024-11-24T01:46:54,866 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T01:46:54,866 INFO [M:0;a67e7d17e593:37581 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T01:46:54,866 INFO [M:0;a67e7d17e593:37581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T01:46:54,867 DEBUG [M:0;a67e7d17e593:37581 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T01:46:54,867 DEBUG [M:0;a67e7d17e593:37581 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T01:46:54,867 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T01:46:54,867 DEBUG [master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732412776068 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.small.0-1732412776068,5,FailOnTimeoutGroup] 2024-11-24T01:46:54,867 DEBUG [master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732412776067 {}] cleaner.HFileCleaner(306): Exit Thread[master/a67e7d17e593:0:becomeActiveMaster-HFileCleaner.large.0-1732412776067,5,FailOnTimeoutGroup] 2024-11-24T01:46:54,868 INFO [M:0;a67e7d17e593:37581 {}] hbase.ChoreService(370): Chore service for: master/a67e7d17e593:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T01:46:54,868 INFO [M:0;a67e7d17e593:37581 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T01:46:54,868 DEBUG [M:0;a67e7d17e593:37581 {}] master.HMaster(1795): Stopping service threads 2024-11-24T01:46:54,868 INFO [M:0;a67e7d17e593:37581 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T01:46:54,869 INFO [M:0;a67e7d17e593:37581 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T01:46:54,870 INFO [M:0;a67e7d17e593:37581 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T01:46:54,870 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T01:46:54,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T01:46:54,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T01:46:54,875 DEBUG [M:0;a67e7d17e593:37581 {}] zookeeper.ZKUtil(347): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T01:46:54,875 WARN [M:0;a67e7d17e593:37581 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T01:46:54,875 INFO [M:0;a67e7d17e593:37581 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/.lastflushedseqids 2024-11-24T01:46:54,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741941_1121 (size=138) 2024-11-24T01:46:54,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741941_1121 (size=138) 2024-11-24T01:46:54,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741941_1121 (size=138) 2024-11-24T01:46:54,887 INFO [M:0;a67e7d17e593:37581 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T01:46:54,887 INFO [M:0;a67e7d17e593:37581 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T01:46:54,887 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T01:46:54,887 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:54,887 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:54,887 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T01:46:54,887 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:54,887 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-24T01:46:54,903 DEBUG [M:0;a67e7d17e593:37581 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/720ad2852fa342399938a03ccbcca374 is 82, key is hbase:meta,,1/info:regioninfo/1732412777200/Put/seqid=0 2024-11-24T01:46:54,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741942_1122 (size=5672) 2024-11-24T01:46:54,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741942_1122 (size=5672) 2024-11-24T01:46:54,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741942_1122 (size=5672) 2024-11-24T01:46:54,909 INFO [M:0;a67e7d17e593:37581 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/720ad2852fa342399938a03ccbcca374 2024-11-24T01:46:54,926 DEBUG [M:0;a67e7d17e593:37581 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be31e10049d64766b3a52a143660f4ae is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732412790788/Put/seqid=0 2024-11-24T01:46:54,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741943_1123 (size=7755) 2024-11-24T01:46:54,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741943_1123 (size=7755) 2024-11-24T01:46:54,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741943_1123 (size=7755) 2024-11-24T01:46:54,932 INFO [M:0;a67e7d17e593:37581 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be31e10049d64766b3a52a143660f4ae 2024-11-24T01:46:54,936 INFO [M:0;a67e7d17e593:37581 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be31e10049d64766b3a52a143660f4ae 2024-11-24T01:46:54,949 DEBUG [M:0;a67e7d17e593:37581 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/670cde40d9d1432482f445cc8dd9e2ac is 69, key is a67e7d17e593,33283,1732412774831/rs:state/1732412776201/Put/seqid=0 2024-11-24T01:46:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741944_1124 (size=5445) 2024-11-24T01:46:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741944_1124 (size=5445) 2024-11-24T01:46:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741944_1124 (size=5445) 2024-11-24T01:46:54,957 INFO [M:0;a67e7d17e593:37581 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/670cde40d9d1432482f445cc8dd9e2ac 2024-11-24T01:46:54,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:54,957 INFO [RS:1;a67e7d17e593:33283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T01:46:54,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33283-0x1016bcea5fe0002, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:54,957 INFO [RS:1;a67e7d17e593:33283 {}] regionserver.HRegionServer(1031): Exiting; stopping=a67e7d17e593,33283,1732412774831; zookeeper connection closed. 2024-11-24T01:46:54,958 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2fdadeb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2fdadeb 2024-11-24T01:46:54,958 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T01:46:54,961 INFO [M:0;a67e7d17e593:37581 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 670cde40d9d1432482f445cc8dd9e2ac 2024-11-24T01:46:54,962 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/720ad2852fa342399938a03ccbcca374 as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/720ad2852fa342399938a03ccbcca374 2024-11-24T01:46:54,967 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/720ad2852fa342399938a03ccbcca374, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T01:46:54,968 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/be31e10049d64766b3a52a143660f4ae as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/be31e10049d64766b3a52a143660f4ae 2024-11-24T01:46:54,972 INFO [M:0;a67e7d17e593:37581 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for be31e10049d64766b3a52a143660f4ae 2024-11-24T01:46:54,972 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/be31e10049d64766b3a52a143660f4ae, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T01:46:54,973 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/670cde40d9d1432482f445cc8dd9e2ac as hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/670cde40d9d1432482f445cc8dd9e2ac 2024-11-24T01:46:54,978 INFO [M:0;a67e7d17e593:37581 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 670cde40d9d1432482f445cc8dd9e2ac 2024-11-24T01:46:54,978 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34675/user/jenkins/test-data/be6caac1-a1a7-126a-c0a5-93fc3d2b6352/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/670cde40d9d1432482f445cc8dd9e2ac, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T01:46:54,979 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false 2024-11-24T01:46:54,981 INFO [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T01:46:54,981 DEBUG [M:0;a67e7d17e593:37581 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732412814887Disabling compacts and flushes for region at 1732412814887Disabling writes for close at 1732412814887Obtaining lock to block concurrent updates at 1732412814887Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732412814887Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1732412814888 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732412814888Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732412814888Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732412814903 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732412814903Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732412814913 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732412814926 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732412814926Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732412814936 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732412814949 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732412814949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fd1622d: reopening flushed file at 1732412814961 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7971624f: reopening flushed file at 1732412814967 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4451cadf: reopening flushed file at 1732412814973 (+6 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false at 1732412814979 (+6 ms)Writing region close event to WAL at 1732412814980 (+1 ms)Closed at 1732412814981 (+1 ms) 2024-11-24T01:46:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33251 is added to blk_1073741830_1006 (size=83606) 2024-11-24T01:46:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39823 is added to blk_1073741830_1006 (size=83606) 2024-11-24T01:46:54,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39381 is added to blk_1073741830_1006 (size=83606) 2024-11-24T01:46:54,984 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T01:46:54,984 INFO [M:0;a67e7d17e593:37581 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T01:46:54,984 INFO [M:0;a67e7d17e593:37581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37581 2024-11-24T01:46:54,984 INFO [M:0;a67e7d17e593:37581 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T01:46:55,099 INFO [M:0;a67e7d17e593:37581 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T01:46:55,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:55,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37581-0x1016bcea5fe0000, quorum=127.0.0.1:60515, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T01:46:55,105 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412794235 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412794235 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732412793879/wal.1732412794235 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T01:46:55,105 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732412785835/wal.1732412785954 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:55,106 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732412809803/wal.1732412810064 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:55,108 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412803728 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412803728 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732412794419/wal.1732412803728 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T01:46:55,109 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732412778896/wal.1732412778993 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:55,111 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804546 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804546 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732412803964/wal.1732412804546 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T01:46:55,112 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732412804729/wal.1732412804908 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T01:46:55,114 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal.1732412778730 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal.1732412778730 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732412778262/wal.1732412778730 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T01:46:55,116 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal.1732412777990 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal.1732412777990 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732412777550/wal.1732412777990 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1410427140_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T01:46:55,121 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:55,123 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T01:46:55,123 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T01:46:55,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T01:46:55,124 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,STOPPED} 2024-11-24T01:46:55,127 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T01:46:55,127 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T01:46:55,127 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T01:46:55,127 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2032936213-172.17.0.2-1732412770132 (Datanode Uuid ed5d09ff-5535-42c9-a78d-b432c1bb6737) service to localhost/127.0.0.1:34675 2024-11-24T01:46:55,129 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data5/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,129 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data6/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,129 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T01:46:55,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:55,132 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T01:46:55,132 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T01:46:55,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T01:46:55,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,STOPPED} 2024-11-24T01:46:55,133 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T01:46:55,133 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T01:46:55,133 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2032936213-172.17.0.2-1732412770132 (Datanode Uuid fc7c409e-68ec-4a95-97d6-5e186e903867) service to localhost/127.0.0.1:34675 2024-11-24T01:46:55,133 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T01:46:55,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data3/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,134 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data4/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,135 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T01:46:55,136 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T01:46:55,137 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T01:46:55,137 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T01:46:55,137 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T01:46:55,137 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,STOPPED} 2024-11-24T01:46:55,138 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T01:46:55,138 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T01:46:55,138 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T01:46:55,138 WARN [BP-2032936213-172.17.0.2-1732412770132 heartbeating to localhost/127.0.0.1:34675 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2032936213-172.17.0.2-1732412770132 (Datanode Uuid ca64ab46-0690-4a7a-86ce-ac2bffc292b2) service to localhost/127.0.0.1:34675 2024-11-24T01:46:55,139 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data1/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,139 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/cluster_6f2169ba-4652-15a1-19b3-0d9b2f328f37/data/data2/current/BP-2032936213-172.17.0.2-1732412770132 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T01:46:55,139 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T01:46:55,146 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T01:46:55,147 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T01:46:55,147 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T01:46:55,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T01:46:55,147 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ef376d0c-88c8-d044-b417-ed1829fa4f7d/hadoop.log.dir/,STOPPED} 2024-11-24T01:46:55,156 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T01:46:55,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down